From 350b611f1ba3b021084ef9627ee96ce661cb5df1 Mon Sep 17 00:00:00 2001 From: p23 Date: Tue, 3 Jun 2025 23:39:20 +0800 Subject: [PATCH] security updates and some change on protobuf --- PictureMaker/default.py | 3 +- PictureMaker/old.py | 3 +- TODO | 18 +++--- backend/api.py | 5 ++ backend/utils/fileProcessor.py | 9 +-- frontend/grpc/protobuf/igapi.proto | 8 +-- frontend/grpc/protobuf/igapi_pb2.py | 22 +++---- frontend/grpc/protobuf/igapi_pb2.pyi | 29 +++++++++ frontend/grpc/protobuf/igapi_pb2_grpc.py | 78 ++++++++++++------------ frontend/grpc/server.py | 31 +++++++--- requirements.txt | 2 +- 11 files changed, 131 insertions(+), 77 deletions(-) create mode 100644 frontend/grpc/protobuf/igapi_pb2.pyi diff --git a/PictureMaker/default.py b/PictureMaker/default.py index be5edd9..4d2f042 100644 --- a/PictureMaker/default.py +++ b/PictureMaker/default.py @@ -2,6 +2,7 @@ import os import hashlib import time from typing import List +import secrets from playwright.sync_api import sync_playwright from jinja2 import Environment, FileSystemLoader, StrictUndefined @@ -67,7 +68,7 @@ def render(post_context:dict) -> tuple[list[str], int]: # screen shot time.sleep(0.1) while True: - filename = TMP + hashlib.sha512( str(time.time()).encode() ).hexdigest() + ".jpg" + filename = TMP + hashlib.sha512( (str(time.time()) + secrets.token_urlsafe(nbytes=24)).encode() ).hexdigest() + ".jpg" filename = os.path.abspath(filename) page.screenshot(path=filename) fnlist.append(filename) diff --git a/PictureMaker/old.py b/PictureMaker/old.py index 814d112..9eda284 100644 --- a/PictureMaker/old.py +++ b/PictureMaker/old.py @@ -1,5 +1,6 @@ import time import hashlib +import secrets import os from PIL import Image, ImageDraw, ImageFont @@ -33,7 +34,7 @@ def gen(context:dict) -> str | None: fill=(0, 0, 0)) # save - filename = TMP + hashlib.sha512( str(time.time()).encode() ).hexdigest() + ".jpg" + filename = TMP + hashlib.sha512( (str(time.time()) + secrets.token_urlsafe(nbytes=24)).encode() ).hexdigest() + ".jpg" img.save(filename) filename = os.path.abspath(filename) diff --git a/TODO b/TODO index f711fbb..a05bf20 100644 --- a/TODO +++ b/TODO @@ -1,18 +1,18 @@ -[ ] Vuln: XSS in PictureMaker.default : jinja2沒開模板轉義 -[ ] Hash只用timestamp當作seed可能不夠(會撞) -[ ] backend.utils.fileProcessor那邊，考慮改善寫法跟加強安全(尤其是考慮關閉管道) - +[ ] 2FA : Merge from sljh niming +[ ] api: ID查IGID，IGID反查ID，批量查詢 +[ ] 隊列本地檔案存儲(持久化) (測試) +[ ] 隊列重試機制 +[ ] 改善Traceback的收集 +[ ] api: 返回錯誤處理紀錄 [ ] 處理因為ig媒體畫面比例固定，但是使用者圖片畫面比例不固定導致的問題 看要不要幫使用者的媒體填充畫面到正確的比例 -[ ] api: ID查IGID，IGID反查ID -[ ] api: 返回錯誤處理紀錄 -[ ] Protobuf 重新定義 -[ ] 改善Traceback的收集 -[ ] 隊列本地檔案存儲 +[好了一半，反正之後還可能擴展] Protobuf 重新定義 [V] 本地儲存ID對IGID表 [V] Webp Support [V] GIF Support [V] 使用者文章太長，溢出換頁機制 +[V] Hash只用timestamp當作seed可能不夠(會撞) +[V] backend.utils.fileProcessor那邊，考慮改善寫法跟加強安全(尤其是考慮關閉管道) [ ] 測試 \ No newline at end of file diff --git a/backend/api.py b/backend/api.py index 2788ffd..ddd9674 100644 --- a/backend/api.py +++ b/backend/api.py @@ -51,6 +51,11 @@ def BACKEND_queue() -> dict: return reply +# search +def BACKEND_search(aid:int | None, igid:str | None) -> dict | None: + return dbhelper.solo_article_fetcher(aid=aid, igid=igid) + + # task: upload def upload(aid:int) -> Tuple[str, int]: # check - visible diff --git a/backend/utils/fileProcessor.py b/backend/utils/fileProcessor.py index 494514c..20bc73c 100644 --- a/backend/utils/fileProcessor.py +++ b/backend/utils/fileProcessor.py @@ -1,10 +1,11 @@ import time import os import io +import secrets from typing import Tuple import subprocess - from hashlib import sha512 + from PIL import Image from pillow_heif import register_heif_opener import ffmpeg @@ -58,14 +59,14 @@ def video_conventor(filename:str, oriFormat:str, binary:bytes) -> int: crf=28, preset='medium' ) - .run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True) + .run_async() #(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True) ) else: process:subprocess.Popen = ( ffmpeg .input(tmpfile, format=oriFormat) .output(filename, format='mp4') - .run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True) + .run_async() #(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True) ) process.wait() @@ -99,7 +100,7 @@ def file_saver(ftype:str, binary:bytes) -> Tuple[str, int]: return "Invalid file type", 1 # 如果不是 IG 本身支援的檔案 -> 轉檔 - filename = sha512( str(time.time()).encode() ).hexdigest() # 暫存檔名稱 + filename = sha512( (str(time.time()) + secrets.token_urlsafe(nbytes=24)).encode() ).hexdigest() # 暫存檔名稱 opt = "" # output file name if not ( ftype == "image/jpg" or ftype == "image/webp" or \ ftype == "video/mp4" ): diff --git a/frontend/grpc/protobuf/igapi.proto b/frontend/grpc/protobuf/igapi.proto index eeff066..d0d3b2f 100644 --- a/frontend/grpc/protobuf/igapi.proto +++ b/frontend/grpc/protobuf/igapi.proto @@ -9,14 +9,14 @@ service IGAPI { rpc delete (Request) returns (Reply) {} - rpc setting (Request) returns (Reply) {} - rpc queue (Request) returns (Reply) {} + + rpc search (Request) returns (Reply) {} } message Request { - int64 code = 1; - repeated string args = 2; + int64 id = 1; + string igid = 2; } message Reply { diff --git a/frontend/grpc/protobuf/igapi_pb2.py b/frontend/grpc/protobuf/igapi_pb2.py index 9c290bc..d5d007a 100644 --- a/frontend/grpc/protobuf/igapi_pb2.py +++ b/frontend/grpc/protobuf/igapi_pb2.py @@ -2,7 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: igapi.proto -# Protobuf Python Version: 5.28.1 +# Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -12,8 +12,8 @@ from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, - 28, - 1, + 29, + 0, '', 'igapi.proto' ) @@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bigapi.proto\"%\n\x07Request\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x03\x12\x0c\n\x04\x61rgs\x18\x02 \x03(\t\"g\n\x05Reply\x12\x0b\n\x03\x65rr\x18\x01 \x01(\x03\x12\"\n\x06result\x18\x02 \x03(\x0b\x32\x12.Reply.ResultEntry\x1a-\n\x0bResultEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x32\xc0\x01\n\x05IGAPI\x12\x1b\n\x05login\x12\x08.Request\x1a\x06.Reply\"\x00\x12\"\n\x0c\x61\x63\x63ount_info\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06upload\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06\x64\x65lete\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1d\n\x07setting\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1b\n\x05queue\x12\x08.Request\x1a\x06.Reply\"\x00\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bigapi.proto\"#\n\x07Request\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0c\n\x04igid\x18\x02 \x01(\t\"g\n\x05Reply\x12\x0b\n\x03\x65rr\x18\x01 \x01(\x03\x12\"\n\x06result\x18\x02 \x03(\x0b\x32\x12.Reply.ResultEntry\x1a-\n\x0bResultEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x32\xbf\x01\n\x05IGAPI\x12\x1b\n\x05login\x12\x08.Request\x1a\x06.Reply\"\x00\x12\"\n\x0c\x61\x63\x63ount_info\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06upload\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06\x64\x65lete\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1b\n\x05queue\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06search\x12\x08.Request\x1a\x06.Reply\"\x00\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -34,11 +34,11 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['_REPLY_RESULTENTRY']._loaded_options = None _globals['_REPLY_RESULTENTRY']._serialized_options = b'8\001' _globals['_REQUEST']._serialized_start=15 - _globals['_REQUEST']._serialized_end=52 - _globals['_REPLY']._serialized_start=54 - _globals['_REPLY']._serialized_end=157 - _globals['_REPLY_RESULTENTRY']._serialized_start=112 - _globals['_REPLY_RESULTENTRY']._serialized_end=157 - _globals['_IGAPI']._serialized_start=160 - _globals['_IGAPI']._serialized_end=352 + _globals['_REQUEST']._serialized_end=50 + _globals['_REPLY']._serialized_start=52 + _globals['_REPLY']._serialized_end=155 + _globals['_REPLY_RESULTENTRY']._serialized_start=110 + _globals['_REPLY_RESULTENTRY']._serialized_end=155 + _globals['_IGAPI']._serialized_start=158 + _globals['_IGAPI']._serialized_end=349 # @@protoc_insertion_point(module_scope) diff --git a/frontend/grpc/protobuf/igapi_pb2.pyi b/frontend/grpc/protobuf/igapi_pb2.pyi new file mode 100644 index 0000000..66b800f --- /dev/null +++ b/frontend/grpc/protobuf/igapi_pb2.pyi @@ -0,0 +1,29 @@ +from google.protobuf.internal import containers as _containers +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional + +DESCRIPTOR: _descriptor.FileDescriptor + +class Request(_message.Message): + __slots__ = ("id", "igid") + ID_FIELD_NUMBER: _ClassVar[int] + IGID_FIELD_NUMBER: _ClassVar[int] + id: int + igid: str + def __init__(self, id: _Optional[int] = ..., igid: _Optional[str] = ...) -> None: ... + +class Reply(_message.Message): + __slots__ = ("err", "result") + class ResultEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + ERR_FIELD_NUMBER: _ClassVar[int] + RESULT_FIELD_NUMBER: _ClassVar[int] + err: int + result: _containers.ScalarMap[str, str] + def __init__(self, err: _Optional[int] = ..., result: _Optional[_Mapping[str, str]] = ...) -> None: ... diff --git a/frontend/grpc/protobuf/igapi_pb2_grpc.py b/frontend/grpc/protobuf/igapi_pb2_grpc.py index 5567161..11a0547 100644 --- a/frontend/grpc/protobuf/igapi_pb2_grpc.py +++ b/frontend/grpc/protobuf/igapi_pb2_grpc.py @@ -3,9 +3,9 @@ import grpc import warnings -from frontend.grpc.protobuf import igapi_pb2 as igapi__pb2 +import frontend.grpc.protobuf.igapi_pb2 as igapi__pb2 -GRPC_GENERATED_VERSION = '1.68.0' +GRPC_GENERATED_VERSION = '1.71.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False @@ -54,13 +54,13 @@ class IGAPIStub(object): request_serializer=igapi__pb2.Request.SerializeToString, response_deserializer=igapi__pb2.Reply.FromString, _registered_method=True) - self.setting = channel.unary_unary( - '/IGAPI/setting', + self.queue = channel.unary_unary( + '/IGAPI/queue', request_serializer=igapi__pb2.Request.SerializeToString, response_deserializer=igapi__pb2.Reply.FromString, _registered_method=True) - self.queue = channel.unary_unary( - '/IGAPI/queue', + self.search = channel.unary_unary( + '/IGAPI/search', request_serializer=igapi__pb2.Request.SerializeToString, response_deserializer=igapi__pb2.Reply.FromString, _registered_method=True) @@ -93,13 +93,13 @@ class IGAPIServicer(object): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def setting(self, request, context): + def queue(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def queue(self, request, context): + def search(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -128,13 +128,13 @@ def add_IGAPIServicer_to_server(servicer, server): request_deserializer=igapi__pb2.Request.FromString, response_serializer=igapi__pb2.Reply.SerializeToString, ), - 'setting': grpc.unary_unary_rpc_method_handler( - servicer.setting, + 'queue': grpc.unary_unary_rpc_method_handler( + servicer.queue, request_deserializer=igapi__pb2.Request.FromString, response_serializer=igapi__pb2.Reply.SerializeToString, ), - 'queue': grpc.unary_unary_rpc_method_handler( - servicer.queue, + 'search': grpc.unary_unary_rpc_method_handler( + servicer.search, request_deserializer=igapi__pb2.Request.FromString, response_serializer=igapi__pb2.Reply.SerializeToString, ), @@ -257,33 +257,6 @@ class IGAPI(object): metadata, _registered_method=True) - @staticmethod - def setting(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/IGAPI/setting', - igapi__pb2.Request.SerializeToString, - igapi__pb2.Reply.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) - @staticmethod def queue(request, target, @@ -310,3 +283,30 @@ class IGAPI(object): timeout, metadata, _registered_method=True) + + @staticmethod + def search(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/IGAPI/search', + igapi__pb2.Request.SerializeToString, + igapi__pb2.Reply.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/frontend/grpc/server.py b/frontend/grpc/server.py index 28f2702..1024b0e 100644 --- a/frontend/grpc/server.py +++ b/frontend/grpc/server.py @@ -42,7 +42,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer): async def upload(self, request: Request, context) -> Reply: grpclog.info("Request: upload") - aid = request.code + aid = request.id res, err = api.upload(aid) if err: return Reply(err=1, result={"error":res}) @@ -52,7 +52,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer): async def delete(self, request: Request, context) -> Reply: grpclog.info("Request: delete") - aid = request.code + aid = request.id res, err = api.delete(aid) if err: return Reply(err=1, result={"error":res}) @@ -66,12 +66,29 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer): return Reply(err=0, result=reply) - async def setting(self, request:Request, context) -> Reply: - # not done - grpclog.info("Request: setting") - return Reply(err=1, result={"error":"Not Done"}) + #async def setting(self, request:Request, context) -> Reply: + # # not done + # grpclog.info("Request: setting") + # return Reply(err=1, result={"error":"Not Done"}) - # get igid with article id + # search 重作 + async def search(self, request:Request, context) -> Reply: + grpclog.info("Request: search") + # search + query_type = "" + if len(request.igid) > 0: # if igid is exist, use it first + reply = api.BACKEND_search(aid=None, igid=request.igid) + query_type = "igid" + else: + reply = api.BACKEND_search(aid=request.id, igid=None) + query_type = "id" + # return + if reply is None: + return Reply(err=1, result={"error":"Not found"}) + else: + reply["id"] = str(reply["id"]) + reply["query"] = query_type + return Reply(err=0, result=reply) # start server diff --git a/requirements.txt b/requirements.txt index 02332cc..88aa71b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ python-magic # frontend.grpc grpcio -protobuf==5.28.3 +protobuf==5.29.0 # PictureMaker playwright