security updates and some change on protobuf

PictureMaker/default.py

@@ -2,6 +2,7 @@ import os
 import hashlib
 import time
 from typing import List
+import secrets
 
 from playwright.sync_api import sync_playwright
 from jinja2 import Environment, FileSystemLoader, StrictUndefined
@@ -67,7 +68,7 @@ def render(post_context:dict) -> tuple[list[str], int]:
             # screen shot
             time.sleep(0.1)
             while True:
-                filename = TMP + hashlib.sha512( str(time.time()).encode() ).hexdigest() + ".jpg"
+                filename = TMP + hashlib.sha512( (str(time.time()) + secrets.token_urlsafe(nbytes=24)).encode() ).hexdigest() + ".jpg"
                 filename = os.path.abspath(filename)
                 page.screenshot(path=filename)
                 fnlist.append(filename)

PictureMaker/old.py

@@ -1,5 +1,6 @@
 import time
 import hashlib
+import secrets
 import os
 
 from PIL import Image, ImageDraw, ImageFont
@@ -33,7 +34,7 @@ def gen(context:dict) -> str | None:
               fill=(0, 0, 0))
     
     # save
-    filename = TMP + hashlib.sha512( str(time.time()).encode() ).hexdigest() + ".jpg"
+    filename = TMP + hashlib.sha512( (str(time.time()) + secrets.token_urlsafe(nbytes=24)).encode() ).hexdigest() + ".jpg"
     img.save(filename)
     filename = os.path.abspath(filename)
     

TODO

@@ -1,18 +1,18 @@
-[ ] Vuln: XSS in PictureMaker.default : jinja2沒開模板轉義
+[ ] 2FA : Merge from sljh niming
-[ ] Hash只用timestamp當作seed可能不夠(會撞)
+[ ] api: ID查IGID，IGID反查ID，批量查詢
-[ ] backend.utils.fileProcessor那邊，考慮改善寫法跟加強安全(尤其是考慮關閉管道)
+[ ] 隊列本地檔案存儲(持久化) (測試)
-
+[ ] 隊列重試機制
+[ ] 改善Traceback的收集
+[ ] api: 返回錯誤處理紀錄
 [ ] 處理因為ig媒體畫面比例固定，但是使用者圖片畫面比例不固定導致的問題
     看要不要幫使用者的媒體填充畫面到正確的比例
-[ ] api: ID查IGID，IGID反查ID
+[好了一半，反正之後還可能擴展] Protobuf 重新定義
-[ ] api: 返回錯誤處理紀錄
-[ ] Protobuf 重新定義
-[ ] 改善Traceback的收集
-[ ] 隊列本地檔案存儲
 
 [V] 本地儲存ID對IGID表
 [V] Webp Support
 [V] GIF Support
 [V] 使用者文章太長，溢出換頁機制
+[V] Hash只用timestamp當作seed可能不夠(會撞)
+[V] backend.utils.fileProcessor那邊，考慮改善寫法跟加強安全(尤其是考慮關閉管道)
 
 [ ] 測試

backend/api.py

@@ -51,6 +51,11 @@ def BACKEND_queue() -> dict:
     return reply
 
 
+# search
+def BACKEND_search(aid:int | None, igid:str | None) -> dict | None:
+    return dbhelper.solo_article_fetcher(aid=aid, igid=igid)
+
+
 # task: upload
 def upload(aid:int) -> Tuple[str, int]:
     # check - visible

backend/utils/fileProcessor.py

@@ -1,10 +1,11 @@
 import time
 import os
 import io
+import secrets
 from typing import Tuple
 import subprocess
-
 from hashlib import sha512
+
 from PIL import Image
 from pillow_heif import register_heif_opener
 import ffmpeg
@@ -58,14 +59,14 @@ def video_conventor(filename:str, oriFormat:str, binary:bytes) -> int:
                     crf=28,
                     preset='medium'
                 )
-                .run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True)
+                .run_async() #(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True)
             )
         else:
             process:subprocess.Popen = (
                 ffmpeg
                 .input(tmpfile, format=oriFormat)
                 .output(filename, format='mp4')
-                .run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True)
+                .run_async() #(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True)
             )
         
         process.wait()
@@ -99,7 +100,7 @@ def file_saver(ftype:str, binary:bytes) -> Tuple[str, int]:
         return "Invalid file type", 1
     
     # 如果不是 IG 本身支援的檔案 -> 轉檔
-    filename = sha512( str(time.time()).encode() ).hexdigest() # 暫存檔名稱
+    filename = sha512( (str(time.time()) + secrets.token_urlsafe(nbytes=24)).encode() ).hexdigest() # 暫存檔名稱
     opt = "" # output file name
     if not ( ftype == "image/jpg" or ftype == "image/webp" or \
             ftype == "video/mp4" ):

frontend/grpc/protobuf/igapi.proto

@@ -9,14 +9,14 @@ service IGAPI {
 
   rpc delete (Request) returns (Reply) {}
 
-  rpc setting (Request) returns (Reply) {}
-
   rpc queue (Request) returns (Reply) {}
+
+  rpc search (Request) returns (Reply) {}
 }
 
 message Request {
-  int64 code = 1;
+  int64 id = 1;
-  repeated string args = 2;
+  string igid = 2;
 }
 
 message Reply {

frontend/grpc/protobuf/igapi_pb2.py

@@ -2,7 +2,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # NO CHECKED-IN PROTOBUF GENCODE
 # source: igapi.proto
-# Protobuf Python Version: 5.28.1
+# Protobuf Python Version: 5.29.0
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -12,8 +12,8 @@ from google.protobuf.internal import builder as _builder
 _runtime_version.ValidateProtobufRuntimeVersion(
     _runtime_version.Domain.PUBLIC,
     5,
-    28,
+    29,
-    1,
+    0,
     '',
     'igapi.proto'
 )
@@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bigapi.proto\"%\n\x07Request\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x03\x12\x0c\n\x04\x61rgs\x18\x02 \x03(\t\"g\n\x05Reply\x12\x0b\n\x03\x65rr\x18\x01 \x01(\x03\x12\"\n\x06result\x18\x02 \x03(\x0b\x32\x12.Reply.ResultEntry\x1a-\n\x0bResultEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x32\xc0\x01\n\x05IGAPI\x12\x1b\n\x05login\x12\x08.Request\x1a\x06.Reply\"\x00\x12\"\n\x0c\x61\x63\x63ount_info\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06upload\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06\x64\x65lete\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1d\n\x07setting\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1b\n\x05queue\x12\x08.Request\x1a\x06.Reply\"\x00\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bigapi.proto\"#\n\x07Request\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0c\n\x04igid\x18\x02 \x01(\t\"g\n\x05Reply\x12\x0b\n\x03\x65rr\x18\x01 \x01(\x03\x12\"\n\x06result\x18\x02 \x03(\x0b\x32\x12.Reply.ResultEntry\x1a-\n\x0bResultEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x32\xbf\x01\n\x05IGAPI\x12\x1b\n\x05login\x12\x08.Request\x1a\x06.Reply\"\x00\x12\"\n\x0c\x61\x63\x63ount_info\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06upload\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06\x64\x65lete\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1b\n\x05queue\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06search\x12\x08.Request\x1a\x06.Reply\"\x00\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -34,11 +34,11 @@ if not _descriptor._USE_C_DESCRIPTORS:
   _globals['_REPLY_RESULTENTRY']._loaded_options = None
   _globals['_REPLY_RESULTENTRY']._serialized_options = b'8\001'
   _globals['_REQUEST']._serialized_start=15
-  _globals['_REQUEST']._serialized_end=52
+  _globals['_REQUEST']._serialized_end=50
-  _globals['_REPLY']._serialized_start=54
+  _globals['_REPLY']._serialized_start=52
-  _globals['_REPLY']._serialized_end=157
+  _globals['_REPLY']._serialized_end=155
-  _globals['_REPLY_RESULTENTRY']._serialized_start=112
+  _globals['_REPLY_RESULTENTRY']._serialized_start=110
-  _globals['_REPLY_RESULTENTRY']._serialized_end=157
+  _globals['_REPLY_RESULTENTRY']._serialized_end=155
-  _globals['_IGAPI']._serialized_start=160
+  _globals['_IGAPI']._serialized_start=158
-  _globals['_IGAPI']._serialized_end=352
+  _globals['_IGAPI']._serialized_end=349
 # @@protoc_insertion_point(module_scope)

frontend/grpc/protobuf/igapi_pb2.pyi

@@ -0,0 +1,29 @@
+from google.protobuf.internal import containers as _containers
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class Request(_message.Message):
+    __slots__ = ("id", "igid")
+    ID_FIELD_NUMBER: _ClassVar[int]
+    IGID_FIELD_NUMBER: _ClassVar[int]
+    id: int
+    igid: str
+    def __init__(self, id: _Optional[int] = ..., igid: _Optional[str] = ...) -> None: ...
+
+class Reply(_message.Message):
+    __slots__ = ("err", "result")
+    class ResultEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    ERR_FIELD_NUMBER: _ClassVar[int]
+    RESULT_FIELD_NUMBER: _ClassVar[int]
+    err: int
+    result: _containers.ScalarMap[str, str]
+    def __init__(self, err: _Optional[int] = ..., result: _Optional[_Mapping[str, str]] = ...) -> None: ...

frontend/grpc/protobuf/igapi_pb2_grpc.py

@@ -3,9 +3,9 @@
 import grpc
 import warnings
 
-from frontend.grpc.protobuf import igapi_pb2 as igapi__pb2
+import frontend.grpc.protobuf.igapi_pb2 as igapi__pb2
 
-GRPC_GENERATED_VERSION = '1.68.0'
+GRPC_GENERATED_VERSION = '1.71.0'
 GRPC_VERSION = grpc.__version__
 _version_not_supported = False
 
@@ -54,13 +54,13 @@ class IGAPIStub(object):
                 request_serializer=igapi__pb2.Request.SerializeToString,
                 response_deserializer=igapi__pb2.Reply.FromString,
                 _registered_method=True)
-        self.setting = channel.unary_unary(
+        self.queue = channel.unary_unary(
-                '/IGAPI/setting',
+                '/IGAPI/queue',
                 request_serializer=igapi__pb2.Request.SerializeToString,
                 response_deserializer=igapi__pb2.Reply.FromString,
                 _registered_method=True)
-        self.queue = channel.unary_unary(
+        self.search = channel.unary_unary(
-                '/IGAPI/queue',
+                '/IGAPI/search',
                 request_serializer=igapi__pb2.Request.SerializeToString,
                 response_deserializer=igapi__pb2.Reply.FromString,
                 _registered_method=True)
@@ -93,13 +93,13 @@ class IGAPIServicer(object):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def setting(self, request, context):
+    def queue(self, request, context):
         """Missing associated documentation comment in .proto file."""
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def queue(self, request, context):
+    def search(self, request, context):
         """Missing associated documentation comment in .proto file."""
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
@@ -128,13 +128,13 @@ def add_IGAPIServicer_to_server(servicer, server):
                     request_deserializer=igapi__pb2.Request.FromString,
                     response_serializer=igapi__pb2.Reply.SerializeToString,
             ),
-            'setting': grpc.unary_unary_rpc_method_handler(
+            'queue': grpc.unary_unary_rpc_method_handler(
-                    servicer.setting,
+                    servicer.queue,
                     request_deserializer=igapi__pb2.Request.FromString,
                     response_serializer=igapi__pb2.Reply.SerializeToString,
             ),
-            'queue': grpc.unary_unary_rpc_method_handler(
+            'search': grpc.unary_unary_rpc_method_handler(
-                    servicer.queue,
+                    servicer.search,
                     request_deserializer=igapi__pb2.Request.FromString,
                     response_serializer=igapi__pb2.Reply.SerializeToString,
             ),
@@ -257,33 +257,6 @@ class IGAPI(object):
             metadata,
             _registered_method=True)
 
-    @staticmethod
-    def setting(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
-        return grpc.experimental.unary_unary(
-            request,
-            target,
-            '/IGAPI/setting',
-            igapi__pb2.Request.SerializeToString,
-            igapi__pb2.Reply.FromString,
-            options,
-            channel_credentials,
-            insecure,
-            call_credentials,
-            compression,
-            wait_for_ready,
-            timeout,
-            metadata,
-            _registered_method=True)
-
     @staticmethod
     def queue(request,
             target,
@@ -310,3 +283,30 @@ class IGAPI(object):
             timeout,
             metadata,
             _registered_method=True)
+
+    @staticmethod
+    def search(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/IGAPI/search',
+            igapi__pb2.Request.SerializeToString,
+            igapi__pb2.Reply.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)

frontend/grpc/server.py

@@ -42,7 +42,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer):
     
     async def upload(self, request: Request, context) -> Reply:
         grpclog.info("Request: upload")
-        aid = request.code
+        aid = request.id
         res, err = api.upload(aid)
         if err:
             return Reply(err=1, result={"error":res})
@@ -52,7 +52,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer):
     
     async def delete(self, request: Request, context) -> Reply:
         grpclog.info("Request: delete")
-        aid = request.code
+        aid = request.id
         res, err = api.delete(aid)
         if err:
             return Reply(err=1, result={"error":res})
@@ -66,12 +66,29 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer):
         return Reply(err=0, result=reply)
     
     
-    async def setting(self, request:Request, context) -> Reply:
+    #async def setting(self, request:Request, context) -> Reply:
-        # not done
+    #    # not done
-        grpclog.info("Request: setting")
+    #    grpclog.info("Request: setting")
-        return Reply(err=1, result={"error":"Not Done"})
+    #    return Reply(err=1, result={"error":"Not Done"})
     
-    # get igid with article id
+    # search 重作
+    async def search(self, request:Request, context) -> Reply:
+        grpclog.info("Request: search")
+        # search
+        query_type = ""
+        if len(request.igid) > 0: # if igid is exist, use it first
+            reply = api.BACKEND_search(aid=None, igid=request.igid)
+            query_type = "igid"
+        else:
+            reply = api.BACKEND_search(aid=request.id, igid=None)
+            query_type = "id"
+        # return
+        if reply is None:
+            return Reply(err=1, result={"error":"Not found"})
+        else:
+            reply["id"] = str(reply["id"])
+            reply["query"] = query_type
+            return Reply(err=0, result=reply)
 
 
 # start server

requirements.txt

@@ -11,7 +11,7 @@ python-magic
 
 # frontend.grpc
 grpcio
-protobuf==5.28.3
+protobuf==5.29.0
 
 # PictureMaker
 playwright