jasinco/niming_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

protobuf

Browse Source
This commit is contained in:
p23 2024-11-25 13:51:50 +00:00
parent d381d1a743
commit 07cb2ac2cc
11 changed files with 165 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app.py
View File

@ -72,3 +72,5 @@ def index():
# app run
if __name__ == "__main__":
app.run(host="0.0.0.0", port=5000, debug=False)
# 檢查ctx跟content的混用(英文單字)

4
blueprints/admin.py
View File

@ -180,8 +180,8 @@ def article_fileget(id:int):
@admin.route('/article/list', methods = ["GET"])
@role_required(["article.read"])
def article_list():
res, code = multi_article_fetcher("admin", request.args.get("start"), request.args.get("count"))
return res, code
res, code = multi_article_fetcher("admin", request.args.get("page"), 30)
return jsonify(res), code
@admin.route("/article/<int:id>", methods=["GET"])
@role_required(["article.read"])

42
blueprints/article.py
View File

@ -7,7 +7,7 @@ from google.protobuf.message import DecodeError
from utils import logger, pgclass, setting_loader
from utils.dbhelper import db, solo_article_fetcher, multi_article_fetcher, solo_file_fetcher, solo_article_remover
from utils.misc import error
from utils.misc import error, error_proto, internal_json2protobuf
from protobuf_files import niming_pb2
"""
@ -25,7 +25,8 @@ article = Blueprint('article', __name__)
# 匿名文列表
@article.route('/list', methods = ["GET"])
def listing():
res, code = multi_article_fetcher("general", request.args.get("start"), request.args.get("count"))
res, code = multi_article_fetcher("general", request.args.get("page"), 30)
res = internal_json2protobuf(res)
return res, code
# 獲取匿名文附檔
@ -71,14 +72,14 @@ def posting():
maxword = opt["Niming_Max_Word"]
# protobuf parse
recv = niming_pb2.DataMessage()
recv = niming_pb2.Post()
try: recv.ParseFromString(request.data)
except DecodeError: return error("Protobuf decode error"), 400
except DecodeError: return error_proto("Protobuf decode error"), 400
# content
ctx = str(recv.ctx)
ctx = str(recv.content)
if len(ctx) == 0 or len(ctx) > maxword: # length check
return error("no content or too many words"), 400
return error_proto("no content or too many words"), 400
# hash
seed = ctx + str(time.time())
@ -92,9 +93,9 @@ def posting():
if not (ref == 0): # 如果ref不是0
# 檢查是不是指向存在的文章
chk = session.query(table).filter(table.id == ref, table.mark == "visible").first()
if chk is None: return error("Invalid Reference"), 400
if chk is None: return error_proto("Invalid Reference"), 400
# 檢查指向的文章是否也是留言
if not(chk.reference is None): return error("Invalid Reference"), 400
if not(chk.reference is None): return error_proto("Invalid Reference"), 400
else:
ref = None
@ -103,15 +104,15 @@ def posting():
# check - size
atts = opt["Attachment_Count"]
sizelimit = opt["Attachment_Size"]
if len(files) > atts: return error("Too many files"), 400
if len(files) > atts: return error_proto("Too many files"), 400
for f in files:
if len(f) <= 0 or len(f) > sizelimit: return error("File size error"), 400
if len(f) <= 0 or len(f) > sizelimit: return error_proto("File size error"), 400
# check - mimetype
allowed_mime = opt["Allowed_MIME"]
for f in files:
mime = magic.Magic(mime=True)
type = mime.from_buffer(f)
if not(type in allowed_mime): return error("File format error"), 400
if not(type in allowed_mime): return error_proto("File format error"), 400
# run processor
ftab = pgclass.SQLfile
for f in files:
@ -135,11 +136,26 @@ def posting():
else: mark = "visible"
# posting
data = table(hash = hash, ctx = ctx, igid = igid, mark = mark, reference = ref, ip = ip)
data = table(hash = hash, content = ctx, igid = igid, mark = mark, reference = ref, ip = ip)
session.add(data)
session.commit()
result, code = solo_article_fetcher(role="owner", key=hash)
# logger
logger.logger("newpost", "New post (id=%d point to %s): %s"%(result["id"], ref, mark))
return result, code
# to protobuf
proto = niming_pb2.PostResponse()
proto.status = niming_pb2.PostStatus.Success
proto.hash = hash
proto.id = int(result["id"])
proto_stres = proto.SerializeToString()
rr = niming_pb2.PostResponse()
rr.ParseFromString(proto_stres)
print(rr.hash)
print(proto_stres)
return proto_stres, code
# 介面全部改成protobuf傳輸
# 檔案傳輸加低畫質版本(縮圖)

37
protobuf_files/niming.proto
View File

@ -1,7 +1,36 @@
syntax = "proto3";
message DataMessage {
string ctx = 1;
int64 ref = 2;
repeated bytes files = 3;
// This is for posting a paragraph.
message Post {
string content = 1;
// reply to a post, like a mail chat.
optional int64 ref = 2;
repeated bytes files = 3;
}
enum PostStatus {
Failed = 0;
Success = 1;
}
// The response of the posting, defining what should return.
message PostResponse {
PostStatus status = 1;
string hash = 2;
uint64 id = 3;
optional string failed_message = 4;
}
message FetchResponse {
message Message {
uint64 id = 1;
string content = 2;
// reply to a post, like a mail chat.
optional uint64 ref = 3;
// request files through /article/file/<id> with MIME type.
// See it as a BLOB url;
repeated uint64 files_id = 4;
}
// Several post info
repeated Message posts = 1;
}

7
protobuf_files/niming.proto.old Normal file
View File

@ -0,0 +1,7 @@
syntax = "proto3";
message DataMessage {
string content = 1;
optional int64 ref = 2;
repeated optional bytes files = 3;
}

37
protobuf_files/niming_pb2.py
View File

@ -1,22 +1,11 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: niming.proto
# Protobuf Python Version: 5.28.3
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
5,
28,
3,
'',
'niming.proto'
)
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
@ -24,13 +13,21 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cniming.proto\"6\n\x0b\x44\x61taMessage\x12\x0b\n\x03\x63tx\x18\x01 \x01(\t\x12\x0b\n\x03ref\x18\x02 \x01(\x03\x12\r\n\x05\x66iles\x18\x03 \x03(\x0c\x62\x06proto3')
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cniming.proto\"@\n\x04Post\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t\x12\x10\n\x03ref\x18\x02 \x01(\x03H\x00\x88\x01\x01\x12\r\n\x05\x66iles\x18\x03 \x03(\x0c\x42\x06\n\x04_ref\"u\n\x0cPostResponse\x12\x1b\n\x06status\x18\x01 \x01(\x0e\x32\x0b.PostStatus\x12\x0c\n\x04hash\x18\x02 \x01(\t\x12\n\n\x02id\x18\x03 \x01(\x04\x12\x1b\n\x0e\x66\x61iled_message\x18\x04 \x01(\tH\x00\x88\x01\x01\x42\x11\n\x0f_failed_message\"\x8a\x01\n\rFetchResponse\x12%\n\x05posts\x18\x01 \x03(\x0b\x32\x16.FetchResponse.Message\x1aR\n\x07Message\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\t\x12\x10\n\x03ref\x18\x03 \x01(\x04H\x00\x88\x01\x01\x12\x10\n\x08\x66iles_id\x18\x04 \x03(\x04\x42\x06\n\x04_ref*%\n\nPostStatus\x12\n\n\x06\x46\x61iled\x10\x00\x12\x0b\n\x07Success\x10\x01\x62\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'niming_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
DESCRIPTOR._loaded_options = None
_globals['_DATAMESSAGE']._serialized_start=16
_globals['_DATAMESSAGE']._serialized_end=70
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'niming_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
_POSTSTATUS._serialized_start=342
_POSTSTATUS._serialized_end=379
_POST._serialized_start=16
_POST._serialized_end=80
_POSTRESPONSE._serialized_start=82
_POSTRESPONSE._serialized_end=199
_FETCHRESPONSE._serialized_start=202
_FETCHRESPONSE._serialized_end=340
_FETCHRESPONSE_MESSAGE._serialized_start=258
_FETCHRESPONSE_MESSAGE._serialized_end=340
# @@protoc_insertion_point(module_scope)

36
protobuf_files/niming_pb2.py.old Normal file
View File

@ -0,0 +1,36 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: niming.proto
# Protobuf Python Version: 5.28.3
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
5,
28,
3,
'',
'niming.proto'
)
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cniming.proto\"6\n\x0b\x44\x61taMessage\x12\x0b\n\x03\x63tx\x18\x01 \x01(\t\x12\x0b\n\x03ref\x18\x02 \x01(\x03\x12\r\n\x05\x66iles\x18\x03 \x03(\x0c\x62\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'niming_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
DESCRIPTOR._loaded_options = None
_globals['_DATAMESSAGE']._serialized_start=16
_globals['_DATAMESSAGE']._serialized_end=70
# @@protoc_insertion_point(module_scope)

2
settings.json
View File

@ -1 +1 @@
{"Check_Before_Post": false, "JWT_Valid_Time": 604800, "Niming_Max_Word": 500, "Attachment_Count": 5, "Attachment_Size": 209715200, "Allowed_MIME": ["image/jpeg", "image/pjpeg", "image/png", "image/heic", "image/heif", "video/mp4", "video/quicktime", "video/hevc", "image/gif", "image/webp"]}
{"Check_Before_Post": false, "JWT_Valid_Time": 604800, "Niming_Max_Word": 500, "Attachment_Count": 5, "Attachment_Size": 209715200, "Allowed_MIME": ["image/jpeg", "image/pjpeg", "image/png", "image/heic", "image/heif", "video/mp4", "video/quicktime", "video/hevc", "image/webp"]}

17
utils/dbhelper.py
View File

@ -6,6 +6,7 @@ from sqlalchemy import desc
from utils import pgclass
from utils.misc import error
from protobuf_files import niming_pb2
class db:
_engine = None
@ -37,7 +38,7 @@ def solo_article_fetcher(role:str, key) -> Tuple[Dict,int]: # admin, owner, gene
if res is None: return {"error":"Post not found"}, 404
# mapping
resfn.update({"id": res.id, "ctx": res.ctx, "igid": res.igid, "mark": res.mark, "reference": res.reference})
resfn.update({"id": res.id, "content": res.content, "igid": res.igid, "mark": res.mark, "reference": res.reference})
if role == "admin": resfn["ip"] = res.ip
elif role == "owner": resfn["hash"] = res.hash
@ -53,13 +54,11 @@ def solo_article_fetcher(role:str, key) -> Tuple[Dict,int]: # admin, owner, gene
return resfn, 200
# 獲取文章列表
def multi_article_fetcher(role:str, start:str, count:str) -> Tuple[Response, int]: # general, admin
def multi_article_fetcher(role:str, page:str, count:int) -> Tuple[List, int]: # general, admin
# checker
if start is None or count is None or \
not start.isdigit() or not count.isdigit():
if page is None or not page.isdigit():
return error("Arguments error"), 400
start = int(start)
count = int(count)
page = int(page)*30
table = pgclass.SQLarticle
ftab = pgclass.SQLfile
@ -71,16 +70,16 @@ def multi_article_fetcher(role:str, start:str, count:str) -> Tuple[Response, int
res = session.query(table).filter(table.mark == "visible", table.reference == None)
elif role == "admin":
res = session.query(table).filter(table.reference == None)
res = res.order_by(desc(table.id)).offset(start).limit(count).all()
res = res.order_by(desc(table.id)).offset(page).limit(count).all()
# mapping
for r in res:
rup = {"id":r.id, "ctx":r.ctx, "igid":r.igid, "created_at":r.created_at, "mark":r.mark}
rup = {"id":r.id, "content":r.content, "igid":r.igid, "created_at":r.created_at, "mark":r.mark, "reference":r.reference}
if role == "admin": rup["ip"] = r.ip # 如果是管理員 多給ip
rup["files"] = [ f[0] for f in session.query(ftab.id).filter(ftab.reference == r.hash).all() ] # 檔案
resfn.append(rup)
return jsonify(resfn), 200
return resfn, 200
# 刪除文章
def solo_article_remover(role:str, hash:str=None, id:int=None) -> Tuple[Dict, int]: # admin, general

27
utils/misc.py
View File

@ -1,4 +1,31 @@
from flask import jsonify, Response
from protobuf_files import niming_pb2
def error(message:str) -> Response:
return jsonify({"error":message})
def error_proto(message:str) -> Response:
proto = niming_pb2.PostResponse()
proto.status = niming_pb2.PostStatus.Failed
proto.hash = ""
proto.id = 0
proto.failed_message = message
return proto.SerializeToString()
def internal_json2protobuf(original:list|dict) -> bytes:
if isinstance(original, dict):
original = [original]
res = niming_pb2.FetchResponse()
for o in original:
ob = niming_pb2.FetchResponse.Message()
ob.id = o["id"]
ob.content = o["content"]
if o["reference"]:
ob.ref = o["reference"]
ob.files_id.extend(o["files"])
res.posts.append(ob)
return res.SerializeToString()

4
utils/pgclass.py
View File

@ -9,14 +9,14 @@ class SQLarticle(Base):
id = Column(BIGINT, primary_key=True)
created_at = Column(TIMESTAMP(timezone=True), server_default=func.now())
hash = Column(String)
ctx = Column(String)
content = Column(String)
igid = Column(String)
mark = Column(String)
ip = Column(String)
reference = Column(BIGINT)
def __repr__(self):
return f"<article(id={self.id}, hash={self.hash}, ctx={self.ctx}, igid={self.igid}, mark={self.mark}, created_at={self.created_at}, ip={self.ip}, reference={self.reference})>"
return f"<article(id={self.id}, hash={self.hash}, content={self.content}, igid={self.igid}, mark={self.mark}, created_at={self.created_at}, ip={self.ip}, reference={self.reference})>"
class SQLlog(Base):
__tablename__ = 'logs'