From f74c7dd7833252f439174882c9f89a9cc8680c38 Mon Sep 17 00:00:00 2001 From: p23 Date: Thu, 14 Nov 2024 18:12:21 +0000 Subject: [PATCH] New Feature: File Upload! I can upload fbk's pictures! Let's Gogit add .! --- README.md | 18 +++++ app.py | 2 + blueprints/admin.py | 3 + blueprints/article.py | 142 ++++++++++++++++++++++++++--------- blueprints/log.py | 2 + protobuf_files/niming.proto | 7 ++ protobuf_files/niming_pb2.py | 36 +++++++++ requirements.txt | 4 +- settings.json | 2 +- utils/pgclass.py | 18 ++++- 10 files changed, 194 insertions(+), 40 deletions(-) create mode 100644 README.md create mode 100644 protobuf_files/niming.proto create mode 100644 protobuf_files/niming_pb2.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..99c188d --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +# Niming Backend + +Prepare: +``` +apt update +apt install libmagic1 libmagic-dev -y + +pip3 install -r requirements.txt +``` + +Run: +``` +python3 app.py +``` + +P23, a Sukonbu + +Shirakami Fubuki is the cutest fox!!! \ No newline at end of file diff --git a/app.py b/app.py index 4a4fdd4..f6991de 100644 --- a/app.py +++ b/app.py @@ -7,6 +7,7 @@ from utils.pgclass import Base # blueprints from blueprints.article import article from blueprints.log import log +from blueprints.admin import admin # Global Variables PG_HOST = os.getenv("PG_HOST") @@ -34,6 +35,7 @@ app.shared_resource = sh # register blueprints app.register_blueprint(article, url_prefix = "/article") app.register_blueprint(log , url_prefix = "/log") +app.register_blueprint(admin , url_prefix = "/admin") # index @app.route("/", methods = ["GET", "POST"]) diff --git a/blueprints/admin.py b/blueprints/admin.py index e69de29..a7f703d 100644 --- a/blueprints/admin.py +++ b/blueprints/admin.py @@ -0,0 +1,3 @@ +from flask import Blueprint + +admin = Blueprint("admin", __name__) \ No newline at end of file diff --git a/blueprints/article.py b/blueprints/article.py index 2d7ef58..f99b199 100644 --- a/blueprints/article.py +++ b/blueprints/article.py @@ -1,18 +1,20 @@ -from flask import Blueprint, current_app, request, jsonify +from flask import Blueprint, current_app, request, jsonify, abort, make_response import hashlib import time +import magic # apt install libmagic1 libmagic-dev -y from utils import logger, pgclass, setting_loader from sqlalchemy.orm import sessionmaker from sqlalchemy import desc +from protobuf_files import niming_pb2 """ TODO: -- Image & Video +- admin (看文，審核文章，刪文，[新增用戶，刪除用戶](用戶管理)，管理平台設定) - IG post -- admin (看文，隱藏文章，刪文，[新增用戶，刪除用戶](用戶管理)) - log 的方式之後要重新設計 - IP Record (deploy之前配合rev proxy) +- 檔案完成，但是再看看要不要讓發文者持sha256存取自己發的文的檔案 """ article = Blueprint('article', __name__) @@ -31,12 +33,16 @@ def listing(): # get ctx table = pgclass.SQLarticle - res = session.query(table.id, table.ctx, table.igid, table.created_at, table.mark).order_by(desc(table.id)).filter(table.mark == 'visible', table.reference == None).offset(rst).limit(count).all() + ftab = pgclass.SQLfile + res = session.query(table.id, table.ctx, table.igid, table.created_at, table.mark, table.hash).order_by(desc(table.id)).filter(table.mark == 'visible', table.reference == None).offset(rst).limit(count).all() # mapping - res = [ {"id":r[0], "ctx":r[1], "igid":r[2], "created_at":r[3], "mark":r[4]} for r in res ] + res = [ {"id":r[0], "ctx":r[1], "igid":r[2], "created_at":r[3], "mark":r[4], + "files": [ f.id for f in session.query(ftab).filter(ftab.reference == r[5]).all() ] } for r in res ] + + session.close() - return jsonify(res) + return jsonify(res), 200 # 獲取指定文章 @article.route("/get/", methods = ["GET"]) @@ -47,21 +53,30 @@ def getarticle(id:int): # get ctx table = pgclass.SQLarticle - res = session.query(table.id, table.ctx, table.igid, table.created_at, table.mark, table.reference).filter(table.id == id).filter(table.mark == 'visible').all() + ftab = pgclass.SQLfile + res = session.query(table.id, table.ctx, table.igid, table.created_at, table.mark, table.reference, table.hash).filter(table.id == id).filter(table.mark == 'visible').all() # mapping resfn = [] for r in res: rd = {"id":r[0], "ctx":r[1], "igid":r[2], "created_at":r[3], "mark":r[4], "reference":r[5]} + # comment comments = session.query(table.id).filter(table.reference == int(r[0]), table.mark == "visible").all() rd["comment"] = [ c[0] for c in comments ] + # files + files = session.query(ftab).filter(ftab.reference == r[6]).all() + rd["files"] = [ f.id for f in files ] resfn.append(rd) - return jsonify(resfn) + session.close() + + return jsonify(resfn), 200 # 上傳文章 / 留言 @article.route("/post", methods = ["POST"]) def posting(): + # ctx -> hash -> reference -> file -> IP -> IG -> mark -> post | -> log + # db db = current_app.shared_resource.engine Session = sessionmaker(bind=db) @@ -70,34 +85,55 @@ def posting(): # loadset opt = setting_loader.loadset() chk_before_post = opt["Check_Before_Post"] + maxword = opt["Niming_Max_Word"] + # data parse + recv = niming_pb2.DataMessage() + recv.ParseFromString(request.data) # content - ctx = request.json["ctx"] - if ctx is None: return "no content" - else: ctx = str(ctx) - - # reference - ref = request.json["ref"] - if not (ref is None): # 如果ref不是null - ref = str(ref) # 轉字串 - if not (ref == ""): # 如果不是空字串 -> 檢查 - if not ref.isdigit(): return "Invalid Reference" # 檢查是不是數字 - # 檢查是不是指向存在的文章 - chk = session.query(table).filter(table.id == ref, table.mark == "visible").first() - if chk is None: return "Invalid Reference" - # 檢查指向的文章是否也是留言 - if not(chk.reference is None): return "Invalid Reference" - else: # 如果是空字串 -> 轉 null object - ref = None - - # IP - ip = request.remote_addr + ctx = str(recv.ctx) # request.json["ctx"] + # length check + if len(ctx) == 0 or len(ctx) > maxword: return "no content or too many words", 400 # hash seed = ctx + str(time.time()) hash = hashlib.sha256(seed.encode()).hexdigest() + # reference + ref = int(recv.ref) # request.json["ref"] + if not (ref == 0): # 如果ref不是0 + # 檢查是不是指向存在的文章 + chk = session.query(table).filter(table.id == ref, table.mark == "visible").first() + if chk is None: return "Invalid Reference", 400 + # 檢查指向的文章是否也是留言 + if not(chk.reference is None): return "Invalid Reference", 400 + else: + ref = None + # file processing + files = recv.files + # check - size + atts = opt["Attachment_Count"] + sizelimit = opt["Attachment_Size"] + if len(files) > atts: return "Too many files", 400 + for f in files: + if len(f) <= 0 or len(f) > sizelimit: return "File size error", 400 + # check - mimetype + allowed_mime = opt["Allowed_MIME"] + for f in files: + mime = magic.Magic(mime=True) + type = mime.from_buffer(f) + if not(type in allowed_mime): return "File format error", 400 + # run processor + ftab = pgclass.SQLfile + for f in files: + mime = magic.Magic(mime=True) + type = mime.from_buffer(f) + fsql = ftab(reference = hash, binary = f, type = type) + session.add(fsql) + + # IP + ip = request.remote_addr # ig posting if chk_before_post: @@ -119,13 +155,16 @@ def posting(): # pg getdata res = session.query(table.id, table.ctx, table.igid, table.created_at, table.mark, table.hash, table.reference).filter(table.hash == hash).all() - res = [ {"id":r[0], "ctx":r[1], "igid":r[2], "created_at":r[3], "mark":r[4], "hash":r[5], "reference":r[6]} for r in res ] + fres = session.query(ftab).filter(ftab.reference == hash).all() + res = [ {"id":r[0], "ctx":r[1], "igid":r[2], "created_at":r[3], "mark":r[4], "hash":r[5], "reference":r[6], + "files": [f.id for f in fres] + } for r in res ] session.close() # logger - logger.logger(db, "newpost", "Insert (id=%d): posts (point to %s) / %s"%(res[0]["id"], ref, mark)) + logger.logger(db, "newpost", "New post (id=%d point to %s): %s"%(res[0]["id"], ref, mark)) - return jsonify(res) + return jsonify(res), 200 # 只有發文者可以看到的獲取指定文章 # 只有發文者可以做到的刪除文章 @@ -135,6 +174,7 @@ def owner_getarticle(sha256:str): Session = sessionmaker(bind=db) session = Session() table = pgclass.SQLarticle + ftab = pgclass.SQLfile # 獲取指定文章 if request.method == "GET": @@ -142,25 +182,57 @@ def owner_getarticle(sha256:str): resfn = [] for r in res: rd = {"id":r[0], "ctx":r[1], "igid":r[2], "created_at":r[3], "mark":r[4], "hash":r[5], "reference":r[6]} + # comments comments = session.query(table.id).filter(table.reference == int(r[0])).all() rd["comment"] = [ c[0] for c in comments ] + # files + files = session.query(ftab).filter(ftab.reference == r[5]).all() + rd["files"] = [ f.id for f in files ] resfn.append(rd) - return jsonify(resfn) - # 刪除指定文章跟他們的留言 + return jsonify(resfn), 200 + # 刪除指定文章跟他們的留言、檔案 elif request.method == "DELETE": rcl = [] res = session.query(table).filter(table.hash == sha256).first() # 本體 resc = session.query(table).filter(table.reference == res.id).all() # 留言 + # 刪除本體檔案 + resf = session.query(ftab).filter(ftab.reference == res.hash).all() + for f in resf: session.delete(f) # 刪留言 for c in resc: rcl.append(c.id) + # 刪留言的檔案 + resf = session.query(ftab).filter(ftab.reference == c.hash).all() + for f in resf: session.delete(f) + # 刪留言 session.delete(c) # 刪本體 session.delete(res) # commit session.commit() # logger - logger.logger(db, "delpost", "Delete (id=%d): posts / comments: %s / last status: %s"%(res.id, str(rcl), res.mark)) + logger.logger(db, "delpost", "Delete post (id=%d with comments %s): last_status=%s"%(res.id, str(rcl), res.mark)) return "OK", 200 - session.close() \ No newline at end of file + session.close() + +# 獲取匿名文附檔 +@article.route("/file/") +def getfile(id:int): + db = current_app.shared_resource.engine + Session = sessionmaker(bind=db) + session = Session() + + table = pgclass.SQLarticle + ftab = pgclass.SQLfile + fres = session.query(ftab).filter(ftab.id == id).first() + if fres is None: return "File not found", 400 # 檢查是否存在 + article = session.query(table).filter(table.hash == fres.reference, table.mark == 'visible').first() + if article is None: return "File not found", 400 # 檢查文章本體是否存在/可以閱覽 + + session.close() + + resp = make_response(fres.binary) + resp.headers.set("Content-Type", fres.type) + + return resp \ No newline at end of file diff --git a/blueprints/log.py b/blueprints/log.py index 21b06a3..8f3b903 100644 --- a/blueprints/log.py +++ b/blueprints/log.py @@ -20,6 +20,7 @@ def listlog(): # get ctx table = pgclass.SQLlog res = session.query(table).order_by(desc(table.id)).offset(rst).limit(count).all() + session.close() # mapping res = [ {"id":r.id, "created_at":r.created_at, "source":r.source, "message":r.message} for r in res ] @@ -37,6 +38,7 @@ def getlog(id): # get ctx table = pgclass.SQLlog res = session.query(table).filter(table.id == id).all() + session.close() # mapping res = [ {"id":r.id, "created_at":r.created_at, "source":r.source, "message":r.message} for r in res ] diff --git a/protobuf_files/niming.proto b/protobuf_files/niming.proto new file mode 100644 index 0000000..4367910 --- /dev/null +++ b/protobuf_files/niming.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +message DataMessage { + string ctx = 1; + int64 ref = 2; + repeated bytes files = 3; +} \ No newline at end of file diff --git a/protobuf_files/niming_pb2.py b/protobuf_files/niming_pb2.py new file mode 100644 index 0000000..de56431 --- /dev/null +++ b/protobuf_files/niming_pb2.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: niming.proto +# Protobuf Python Version: 5.28.3 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 5, + 28, + 3, + '', + 'niming.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cniming.proto\"6\n\x0b\x44\x61taMessage\x12\x0b\n\x03\x63tx\x18\x01 \x01(\t\x12\x0b\n\x03ref\x18\x02 \x01(\x03\x12\r\n\x05\x66iles\x18\x03 \x03(\x0c\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'niming_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_DATAMESSAGE']._serialized_start=16 + _globals['_DATAMESSAGE']._serialized_end=70 +# @@protoc_insertion_point(module_scope) diff --git a/requirements.txt b/requirements.txt index eade94c..2e05a63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ sqlalchemy flask pyjwt -psycopg2 \ No newline at end of file +psycopg2 +protobuf==5.28.3 +python-magic \ No newline at end of file diff --git a/settings.json b/settings.json index 8dd40b6..366c5a6 100644 --- a/settings.json +++ b/settings.json @@ -1 +1 @@ -{"Check_Before_Post":false} \ No newline at end of file +{"Check_Before_Post":false, "Niming_Max_Word":500, "Attachment_Count":5, "Attachment_Size":209715200, "Allowed_MIME":["image/jpeg", "image/pjpeg", "image/png", "image/heic", "image/heif", "video/mp4", "video/quicktime", "video/hevc", "image/gif", "image/webp"]} \ No newline at end of file diff --git a/utils/pgclass.py b/utils/pgclass.py index 47d52b8..cb4213a 100644 --- a/utils/pgclass.py +++ b/utils/pgclass.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, Integer, String, TIMESTAMP, func, BIGINT +from sqlalchemy import Column, Integer, String, TIMESTAMP, func, BIGINT, LargeBinary from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() @@ -21,10 +21,22 @@ class SQLarticle(Base): class SQLlog(Base): __tablename__ = 'logs' - id = Column(Integer, primary_key=True) + id = Column(BIGINT, primary_key=True) created_at = Column(TIMESTAMP(timezone=True), server_default=func.now()) message = Column(String) source = Column(String) def __repr__(self): - return f"" \ No newline at end of file + return f"" + +class SQLfile(Base): + __tablename__ = 'files' + + id = Column(BIGINT, primary_key=True) + created_at = Column(TIMESTAMP(timezone=True), server_default=func.now()) + type = Column(String) + reference = Column(String) + binary = Column(LargeBinary) + + def __repr__(self): + return f"" \ No newline at end of file