p23/niming_igapi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

security updates and some change on protobuf

Browse source
This commit is contained in:
p23 2025-06-03 23:39:20 +08:00
parent 1bc1497113
commit 350b611f1b
11 changed files with 131 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
PictureMaker/default.py
View file

@ -2,6 +2,7 @@ import os
import hashlib
import time
from typing import List
import secrets
from playwright.sync_api import sync_playwright
from jinja2 import Environment, FileSystemLoader, StrictUndefined
@ -67,7 +68,7 @@ def render(post_context:dict) -> tuple[list[str], int]:
# screen shot
time.sleep(0.1)
while True:
filename = TMP + hashlib.sha512( str(time.time()).encode() ).hexdigest() + ".jpg"
filename = TMP + hashlib.sha512( (str(time.time()) + secrets.token_urlsafe(nbytes=24)).encode() ).hexdigest() + ".jpg"
filename = os.path.abspath(filename)
page.screenshot(path=filename)
fnlist.append(filename)

3
PictureMaker/old.py
View file

@ -1,5 +1,6 @@
import time
import hashlib
import secrets
import os
from PIL import Image, ImageDraw, ImageFont
@ -33,7 +34,7 @@ def gen(context:dict) -> str | None:
fill=(0, 0, 0))
# save
filename = TMP + hashlib.sha512( str(time.time()).encode() ).hexdigest() + ".jpg"
filename = TMP + hashlib.sha512( (str(time.time()) + secrets.token_urlsafe(nbytes=24)).encode() ).hexdigest() + ".jpg"
img.save(filename)
filename = os.path.abspath(filename)

18
TODO
View file

@ -1,18 +1,18 @@
[ ] Vuln: XSS in PictureMaker.default : jinja2沒開模板轉義
[ ] Hash只用timestamp當作seed可能不夠(會撞)
[ ] backend.utils.fileProcessor那邊，考慮改善寫法跟加強安全(尤其是考慮關閉管道)
[ ] 2FA : Merge from sljh niming
[ ] api: ID查IGID，IGID反查ID，批量查詢
[ ] 隊列本地檔案存儲(持久化) (測試)
[ ] 隊列重試機制
[ ] 改善Traceback的收集
[ ] api: 返回錯誤處理紀錄
[ ] 處理因為ig媒體畫面比例固定，但是使用者圖片畫面比例不固定導致的問題
看要不要幫使用者的媒體填充畫面到正確的比例
[ ] api: ID查IGID，IGID反查ID
[ ] api: 返回錯誤處理紀錄
[ ] Protobuf 重新定義
[ ] 改善Traceback的收集
[ ] 隊列本地檔案存儲
[好了一半，反正之後還可能擴展] Protobuf 重新定義
[V] 本地儲存ID對IGID表
[V] Webp Support
[V] GIF Support
[V] 使用者文章太長，溢出換頁機制
[V] Hash只用timestamp當作seed可能不夠(會撞)
[V] backend.utils.fileProcessor那邊，考慮改善寫法跟加強安全(尤其是考慮關閉管道)
[ ] 測試

5
backend/api.py
View file

@ -51,6 +51,11 @@ def BACKEND_queue() -> dict:
return reply
# search
def BACKEND_search(aid:int | None, igid:str | None) -> dict | None:
return dbhelper.solo_article_fetcher(aid=aid, igid=igid)
# task: upload
def upload(aid:int) -> Tuple[str, int]:
# check - visible

9
backend/utils/fileProcessor.py
View file

@ -1,10 +1,11 @@
import time
import os
import io
import secrets
from typing import Tuple
import subprocess
from hashlib import sha512
from PIL import Image
from pillow_heif import register_heif_opener
import ffmpeg
@ -58,14 +59,14 @@ def video_conventor(filename:str, oriFormat:str, binary:bytes) -> int:
crf=28,
preset='medium'
)
.run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True)
.run_async() #(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True)
)
else:
process:subprocess.Popen = (
ffmpeg
.input(tmpfile, format=oriFormat)
.output(filename, format='mp4')
.run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True)
.run_async() #(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True)
)
process.wait()
@ -99,7 +100,7 @@ def file_saver(ftype:str, binary:bytes) -> Tuple[str, int]:
return "Invalid file type", 1
# 如果不是 IG 本身支援的檔案 -> 轉檔
filename = sha512( str(time.time()).encode() ).hexdigest() # 暫存檔名稱
filename = sha512( (str(time.time()) + secrets.token_urlsafe(nbytes=24)).encode() ).hexdigest() # 暫存檔名稱
opt = "" # output file name
if not ( ftype == "image/jpg" or ftype == "image/webp" or \
ftype == "video/mp4" ):

8
frontend/grpc/protobuf/igapi.proto
View file

@ -9,14 +9,14 @@ service IGAPI {
rpc delete (Request) returns (Reply) {}
rpc setting (Request) returns (Reply) {}
rpc queue (Request) returns (Reply) {}
rpc search (Request) returns (Reply) {}
}
message Request {
int64 code = 1;
repeated string args = 2;
int64 id = 1;
string igid = 2;
}
message Reply {

22
frontend/grpc/protobuf/igapi_pb2.py
View file

@ -2,7 +2,7 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: igapi.proto
# Protobuf Python Version: 5.28.1
# Protobuf Python Version: 5.29.0
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
@ -12,8 +12,8 @@ from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
5,
28,
1,
29,
0,
'',
'igapi.proto'
)
@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bigapi.proto\"%\n\x07Request\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x03\x12\x0c\n\x04\x61rgs\x18\x02 \x03(\t\"g\n\x05Reply\x12\x0b\n\x03\x65rr\x18\x01 \x01(\x03\x12\"\n\x06result\x18\x02 \x03(\x0b\x32\x12.Reply.ResultEntry\x1a-\n\x0bResultEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x32\xc0\x01\n\x05IGAPI\x12\x1b\n\x05login\x12\x08.Request\x1a\x06.Reply\"\x00\x12\"\n\x0c\x61\x63\x63ount_info\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06upload\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06\x64\x65lete\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1d\n\x07setting\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1b\n\x05queue\x12\x08.Request\x1a\x06.Reply\"\x00\x62\x06proto3')
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bigapi.proto\"#\n\x07Request\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0c\n\x04igid\x18\x02 \x01(\t\"g\n\x05Reply\x12\x0b\n\x03\x65rr\x18\x01 \x01(\x03\x12\"\n\x06result\x18\x02 \x03(\x0b\x32\x12.Reply.ResultEntry\x1a-\n\x0bResultEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x32\xbf\x01\n\x05IGAPI\x12\x1b\n\x05login\x12\x08.Request\x1a\x06.Reply\"\x00\x12\"\n\x0c\x61\x63\x63ount_info\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06upload\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06\x64\x65lete\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1b\n\x05queue\x12\x08.Request\x1a\x06.Reply\"\x00\x12\x1c\n\x06search\x12\x08.Request\x1a\x06.Reply\"\x00\x62\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@ -34,11 +34,11 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals['_REPLY_RESULTENTRY']._loaded_options = None
_globals['_REPLY_RESULTENTRY']._serialized_options = b'8\001'
_globals['_REQUEST']._serialized_start=15
_globals['_REQUEST']._serialized_end=52
_globals['_REPLY']._serialized_start=54
_globals['_REPLY']._serialized_end=157
_globals['_REPLY_RESULTENTRY']._serialized_start=112
_globals['_REPLY_RESULTENTRY']._serialized_end=157
_globals['_IGAPI']._serialized_start=160
_globals['_IGAPI']._serialized_end=352
_globals['_REQUEST']._serialized_end=50
_globals['_REPLY']._serialized_start=52
_globals['_REPLY']._serialized_end=155
_globals['_REPLY_RESULTENTRY']._serialized_start=110
_globals['_REPLY_RESULTENTRY']._serialized_end=155
_globals['_IGAPI']._serialized_start=158
_globals['_IGAPI']._serialized_end=349
# @@protoc_insertion_point(module_scope)

29
frontend/grpc/protobuf/igapi_pb2.pyi Normal file
View file

@ -0,0 +1,29 @@
from google.protobuf.internal import containers as _containers
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional
DESCRIPTOR: _descriptor.FileDescriptor
class Request(_message.Message):
__slots__ = ("id", "igid")
ID_FIELD_NUMBER: _ClassVar[int]
IGID_FIELD_NUMBER: _ClassVar[int]
id: int
igid: str
def __init__(self, id: _Optional[int] = ..., igid: _Optional[str] = ...) -> None: ...
class Reply(_message.Message):
__slots__ = ("err", "result")
class ResultEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
VALUE_FIELD_NUMBER: _ClassVar[int]
key: str
value: str
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
ERR_FIELD_NUMBER: _ClassVar[int]
RESULT_FIELD_NUMBER: _ClassVar[int]
err: int
result: _containers.ScalarMap[str, str]
def __init__(self, err: _Optional[int] = ..., result: _Optional[_Mapping[str, str]] = ...) -> None: ...

78
frontend/grpc/protobuf/igapi_pb2_grpc.py
View file

@ -3,9 +3,9 @@
import grpc
import warnings
from frontend.grpc.protobuf import igapi_pb2 as igapi__pb2
import frontend.grpc.protobuf.igapi_pb2 as igapi__pb2
GRPC_GENERATED_VERSION = '1.68.0'
GRPC_GENERATED_VERSION = '1.71.0'
GRPC_VERSION = grpc.__version__
_version_not_supported = False
@ -54,13 +54,13 @@ class IGAPIStub(object):
request_serializer=igapi__pb2.Request.SerializeToString,
response_deserializer=igapi__pb2.Reply.FromString,
_registered_method=True)
self.setting = channel.unary_unary(
'/IGAPI/setting',
self.queue = channel.unary_unary(
'/IGAPI/queue',
request_serializer=igapi__pb2.Request.SerializeToString,
response_deserializer=igapi__pb2.Reply.FromString,
_registered_method=True)
self.queue = channel.unary_unary(
'/IGAPI/queue',
self.search = channel.unary_unary(
'/IGAPI/search',
request_serializer=igapi__pb2.Request.SerializeToString,
response_deserializer=igapi__pb2.Reply.FromString,
_registered_method=True)
@ -93,13 +93,13 @@ class IGAPIServicer(object):
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def setting(self, request, context):
def queue(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def queue(self, request, context):
def search(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
@ -128,13 +128,13 @@ def add_IGAPIServicer_to_server(servicer, server):
request_deserializer=igapi__pb2.Request.FromString,
response_serializer=igapi__pb2.Reply.SerializeToString,
),
'setting': grpc.unary_unary_rpc_method_handler(
servicer.setting,
'queue': grpc.unary_unary_rpc_method_handler(
servicer.queue,
request_deserializer=igapi__pb2.Request.FromString,
response_serializer=igapi__pb2.Reply.SerializeToString,
),
'queue': grpc.unary_unary_rpc_method_handler(
servicer.queue,
'search': grpc.unary_unary_rpc_method_handler(
servicer.search,
request_deserializer=igapi__pb2.Request.FromString,
response_serializer=igapi__pb2.Reply.SerializeToString,
),
@ -257,33 +257,6 @@ class IGAPI(object):
metadata,
_registered_method=True)
@staticmethod
def setting(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(
request,
target,
'/IGAPI/setting',
igapi__pb2.Request.SerializeToString,
igapi__pb2.Reply.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True)
@staticmethod
def queue(request,
target,
@ -310,3 +283,30 @@ class IGAPI(object):
timeout,
metadata,
_registered_method=True)
@staticmethod
def search(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(
request,
target,
'/IGAPI/search',
igapi__pb2.Request.SerializeToString,
igapi__pb2.Reply.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True)

31
frontend/grpc/server.py
View file

@ -42,7 +42,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer):
async def upload(self, request: Request, context) -> Reply:
grpclog.info("Request: upload")
aid = request.code
aid = request.id
res, err = api.upload(aid)
if err:
return Reply(err=1, result={"error":res})
@ -52,7 +52,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer):
async def delete(self, request: Request, context) -> Reply:
grpclog.info("Request: delete")
aid = request.code
aid = request.id
res, err = api.delete(aid)
if err:
return Reply(err=1, result={"error":res})
@ -66,12 +66,29 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer):
return Reply(err=0, result=reply)
async def setting(self, request:Request, context) -> Reply:
# not done
grpclog.info("Request: setting")
return Reply(err=1, result={"error":"Not Done"})
#async def setting(self, request:Request, context) -> Reply:
# # not done
# grpclog.info("Request: setting")
# return Reply(err=1, result={"error":"Not Done"})
# get igid with article id
# search 重作
async def search(self, request:Request, context) -> Reply:
grpclog.info("Request: search")
# search
query_type = ""
if len(request.igid) > 0: # if igid is exist, use it first
reply = api.BACKEND_search(aid=None, igid=request.igid)
query_type = "igid"
else:
reply = api.BACKEND_search(aid=request.id, igid=None)
query_type = "id"
# return
if reply is None:
return Reply(err=1, result={"error":"Not found"})
else:
reply["id"] = str(reply["id"])
reply["query"] = query_type
return Reply(err=0, result=reply)
# start server

2
requirements.txt
View file

@ -11,7 +11,7 @@ python-magic
# frontend.grpc
grpcio
protobuf==5.28.3
protobuf==5.29.0
# PictureMaker
playwright