New Feature: File Upload! I can upload fbk's pictures! Let's Gogit add .!

README.md

@@ -0,0 +1,18 @@
+# Niming Backend
+
+Prepare:
+```
+apt update
+apt install libmagic1 libmagic-dev -y
+
+pip3 install -r requirements.txt
+```
+
+Run:
+```
+python3 app.py
+```
+
+P23, a Sukonbu
+
+Shirakami Fubuki is the cutest fox!!!

app.py

@@ -7,6 +7,7 @@ from utils.pgclass import Base
 # blueprints
 from blueprints.article import article
 from blueprints.log     import log
+from blueprints.admin   import admin
 
 # Global Variables
 PG_HOST = os.getenv("PG_HOST")
@@ -34,6 +35,7 @@ app.shared_resource = sh
 # register blueprints
 app.register_blueprint(article, url_prefix = "/article")
 app.register_blueprint(log    , url_prefix = "/log")
+app.register_blueprint(admin  , url_prefix = "/admin")
 
 # index
 @app.route("/", methods = ["GET", "POST"])

blueprints/admin.py

@@ -0,0 +1,3 @@
+from flask import Blueprint
+
+admin = Blueprint("admin", __name__)

blueprints/article.py

@@ -1,18 +1,20 @@
-from flask import Blueprint, current_app, request, jsonify
+from flask import Blueprint, current_app, request, jsonify, abort, make_response
 import hashlib
 import time
+import magic # apt install libmagic1 libmagic-dev -y
 from utils import logger, pgclass, setting_loader
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy import desc
+from protobuf_files import niming_pb2
 
 """
 TODO:
-- Image & Video
+- admin (看文，審核文章，刪文，[新增用戶，刪除用戶](用戶管理)，管理平台設定)
 - IG post
-- admin (看文，隱藏文章，刪文，[新增用戶，刪除用戶](用戶管理))
 
 - log 的方式之後要重新設計
 - IP Record (deploy之前配合rev proxy)
+- 檔案完成，但是再看看要不要讓發文者持sha256存取自己發的文的檔案
 """
 
 article = Blueprint('article', __name__)
@@ -31,12 +33,16 @@ def listing():
 
     # get ctx
     table = pgclass.SQLarticle
-    res = session.query(table.id, table.ctx, table.igid, table.created_at, table.mark).order_by(desc(table.id)).filter(table.mark == 'visible', table.reference == None).offset(rst).limit(count).all()
+    ftab = pgclass.SQLfile
+    res = session.query(table.id, table.ctx, table.igid, table.created_at, table.mark, table.hash).order_by(desc(table.id)).filter(table.mark == 'visible', table.reference == None).offset(rst).limit(count).all()
 
     # mapping
-    res = [ {"id":r[0], "ctx":r[1], "igid":r[2], "created_at":r[3], "mark":r[4]} for r in res ]
+    res = [ {"id":r[0], "ctx":r[1], "igid":r[2], "created_at":r[3], "mark":r[4],
+             "files": [ f.id for f in session.query(ftab).filter(ftab.reference == r[5]).all() ] } for r in res ]
+    
+    session.close()
 
-    return jsonify(res)
+    return jsonify(res), 200
 
 # 獲取指定文章
 @article.route("/get/<int:id>", methods = ["GET"])
@@ -47,21 +53,30 @@ def getarticle(id:int):
 
     # get ctx
     table = pgclass.SQLarticle
-    res = session.query(table.id, table.ctx, table.igid, table.created_at, table.mark, table.reference).filter(table.id == id).filter(table.mark == 'visible').all()
+    ftab  = pgclass.SQLfile
+    res = session.query(table.id, table.ctx, table.igid, table.created_at, table.mark, table.reference, table.hash).filter(table.id == id).filter(table.mark == 'visible').all()
 
     # mapping
     resfn = []
     for r in res:
         rd = {"id":r[0], "ctx":r[1], "igid":r[2], "created_at":r[3], "mark":r[4], "reference":r[5]}
+        # comment
         comments = session.query(table.id).filter(table.reference == int(r[0]), table.mark == "visible").all()
         rd["comment"] = [ c[0] for c in comments ]
+        # files
+        files = session.query(ftab).filter(ftab.reference == r[6]).all()
+        rd["files"] = [ f.id for f in files ]
         resfn.append(rd)
 
-    return jsonify(resfn)
+    session.close()
+
+    return jsonify(resfn), 200
 
 # 上傳文章 / 留言
 @article.route("/post", methods = ["POST"])
 def posting():
+    # ctx -> hash -> reference -> file -> IP -> IG -> mark -> post | -> log
+
     # db
     db = current_app.shared_resource.engine
     Session = sessionmaker(bind=db)
@@ -70,34 +85,55 @@ def posting():
     # loadset
     opt = setting_loader.loadset()
     chk_before_post = opt["Check_Before_Post"]
+    maxword = opt["Niming_Max_Word"]
+    # data parse
+    recv = niming_pb2.DataMessage()
+    recv.ParseFromString(request.data)
 
     # content
-    ctx = request.json["ctx"]
+    ctx = str(recv.ctx) # request.json["ctx"]
-    if ctx is None: return "no content"
+    # length check
-    else: ctx = str(ctx)
+    if len(ctx) == 0 or len(ctx) > maxword: return "no content or too many words", 400
-
-    # reference
-    ref = request.json["ref"]
-    if not (ref is None):   # 如果ref不是null
-        ref = str(ref)      # 轉字串
-        if not (ref == ""): # 如果不是空字串 -> 檢查
-            if not ref.isdigit(): return "Invalid Reference" # 檢查是不是數字
-            # 檢查是不是指向存在的文章
-            chk = session.query(table).filter(table.id == ref, table.mark == "visible").first()
-            if chk is None: return "Invalid Reference"
-            # 檢查指向的文章是否也是留言
-            if not(chk.reference is None): return "Invalid Reference"
-        else:               # 如果是空字串 -> 轉 null object
-            ref = None
-    
-    # IP
-    ip = request.remote_addr
 
     # hash
     seed = ctx + str(time.time())
     hash = hashlib.sha256(seed.encode()).hexdigest()
 
+    # reference
+    ref = int(recv.ref) # request.json["ref"]
+    if not (ref == 0):   # 如果ref不是0
+        # 檢查是不是指向存在的文章
+        chk = session.query(table).filter(table.id == ref, table.mark == "visible").first()
+        if chk is None: return "Invalid Reference", 400
+        # 檢查指向的文章是否也是留言
+        if not(chk.reference is None): return "Invalid Reference", 400
+    else:
+        ref = None
+    
     # file processing
+    files = recv.files
+    # check - size
+    atts = opt["Attachment_Count"]
+    sizelimit = opt["Attachment_Size"]
+    if len(files) > atts: return "Too many files", 400
+    for f in files:
+        if len(f) <= 0 or len(f) > sizelimit: return "File size error", 400
+    # check - mimetype
+    allowed_mime = opt["Allowed_MIME"]
+    for f in files:
+        mime = magic.Magic(mime=True)
+        type = mime.from_buffer(f)
+        if not(type in allowed_mime): return "File format error", 400
+    # run processor
+    ftab = pgclass.SQLfile
+    for f in files:
+        mime = magic.Magic(mime=True)
+        type = mime.from_buffer(f)
+        fsql = ftab(reference = hash, binary = f, type = type)
+        session.add(fsql)
+
+    # IP
+    ip = request.remote_addr
 
     # ig posting
     if chk_before_post:
@@ -119,13 +155,16 @@ def posting():
 
     # pg getdata
     res = session.query(table.id, table.ctx, table.igid, table.created_at, table.mark, table.hash, table.reference).filter(table.hash == hash).all()
-    res = [ {"id":r[0], "ctx":r[1], "igid":r[2], "created_at":r[3], "mark":r[4], "hash":r[5], "reference":r[6]} for r in res ]
+    fres = session.query(ftab).filter(ftab.reference == hash).all()
+    res = [ {"id":r[0], "ctx":r[1], "igid":r[2], "created_at":r[3], "mark":r[4], "hash":r[5], "reference":r[6],
+             "files": [f.id for f in fres]
+             } for r in res ]
     session.close()
 
     # logger
-    logger.logger(db, "newpost", "Insert (id=%d): posts (point to %s) / %s"%(res[0]["id"], ref, mark))
+    logger.logger(db, "newpost", "New post (id=%d point to %s): %s"%(res[0]["id"], ref, mark))
     
-    return jsonify(res)
+    return jsonify(res), 200
 
 # 只有發文者可以看到的獲取指定文章
 # 只有發文者可以做到的刪除文章
@@ -135,6 +174,7 @@ def owner_getarticle(sha256:str):
     Session = sessionmaker(bind=db)
     session = Session()
     table = pgclass.SQLarticle
+    ftab = pgclass.SQLfile
 
     # 獲取指定文章
     if request.method == "GET":
@@ -142,25 +182,57 @@ def owner_getarticle(sha256:str):
         resfn = []
         for r in res:
             rd = {"id":r[0], "ctx":r[1], "igid":r[2], "created_at":r[3], "mark":r[4], "hash":r[5], "reference":r[6]}
+            # comments
             comments = session.query(table.id).filter(table.reference == int(r[0])).all()
             rd["comment"] = [ c[0] for c in comments ]
+            # files
+            files = session.query(ftab).filter(ftab.reference == r[5]).all()
+            rd["files"] = [ f.id for f in files ]
             resfn.append(rd)
-        return jsonify(resfn)
+        return jsonify(resfn), 200
-    # 刪除指定文章跟他們的留言
+    # 刪除指定文章跟他們的留言、檔案
     elif request.method == "DELETE":
         rcl = []
         res = session.query(table).filter(table.hash == sha256).first()     # 本體
         resc = session.query(table).filter(table.reference == res.id).all() # 留言
+        # 刪除本體檔案
+        resf = session.query(ftab).filter(ftab.reference == res.hash).all()
+        for f in resf: session.delete(f)
         # 刪留言
         for c in resc:
             rcl.append(c.id)
+            # 刪留言的檔案
+            resf = session.query(ftab).filter(ftab.reference == c.hash).all()
+            for f in resf: session.delete(f)
+            # 刪留言
             session.delete(c)
         # 刪本體
         session.delete(res)
         # commit
         session.commit()
         # logger
-        logger.logger(db, "delpost", "Delete (id=%d): posts / comments: %s / last status: %s"%(res.id, str(rcl), res.mark))
+        logger.logger(db, "delpost", "Delete post (id=%d with comments %s): last_status=%s"%(res.id, str(rcl), res.mark))
         return "OK", 200
 
-    session.close()
+    session.close()
+
+# 獲取匿名文附檔
+@article.route("/file/<int:id>")
+def getfile(id:int):
+    db = current_app.shared_resource.engine
+    Session = sessionmaker(bind=db)
+    session = Session()
+
+    table = pgclass.SQLarticle
+    ftab  = pgclass.SQLfile
+    fres = session.query(ftab).filter(ftab.id == id).first()
+    if fres is None: return "File not found", 400       # 檢查是否存在
+    article = session.query(table).filter(table.hash == fres.reference, table.mark == 'visible').first()
+    if article is None: return "File not found", 400    # 檢查文章本體是否存在/可以閱覽
+
+    session.close()
+
+    resp = make_response(fres.binary)
+    resp.headers.set("Content-Type", fres.type)
+    
+    return resp

blueprints/log.py

@@ -20,6 +20,7 @@ def listlog():
     # get ctx
     table = pgclass.SQLlog
     res = session.query(table).order_by(desc(table.id)).offset(rst).limit(count).all()
+    session.close()
 
     # mapping
     res = [ {"id":r.id, "created_at":r.created_at, "source":r.source, "message":r.message} for r in res ]
@@ -37,6 +38,7 @@ def getlog(id):
     # get ctx
     table = pgclass.SQLlog
     res = session.query(table).filter(table.id == id).all()
+    session.close()
 
     # mapping
     res = [ {"id":r.id, "created_at":r.created_at, "source":r.source, "message":r.message} for r in res ]

protobuf_files/niming.proto

@@ -0,0 +1,7 @@
+syntax = "proto3";
+
+message DataMessage {
+    string ctx = 1;
+    int64 ref = 2;
+    repeated bytes files = 3;
+}

protobuf_files/niming_pb2.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: niming.proto
+# Protobuf Python Version: 5.28.3
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_runtime_version.ValidateProtobufRuntimeVersion(
+    _runtime_version.Domain.PUBLIC,
+    5,
+    28,
+    3,
+    '',
+    'niming.proto'
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cniming.proto\"6\n\x0b\x44\x61taMessage\x12\x0b\n\x03\x63tx\x18\x01 \x01(\t\x12\x0b\n\x03ref\x18\x02 \x01(\x03\x12\r\n\x05\x66iles\x18\x03 \x03(\x0c\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'niming_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+  DESCRIPTOR._loaded_options = None
+  _globals['_DATAMESSAGE']._serialized_start=16
+  _globals['_DATAMESSAGE']._serialized_end=70
+# @@protoc_insertion_point(module_scope)

requirements.txt

@@ -1,4 +1,6 @@
 sqlalchemy
 flask
 pyjwt
-psycopg2
+psycopg2
+protobuf==5.28.3
+python-magic

settings.json

@@ -1 +1 @@
-{"Check_Before_Post":false}
+{"Check_Before_Post":false, "Niming_Max_Word":500, "Attachment_Count":5, "Attachment_Size":209715200, "Allowed_MIME":["image/jpeg", "image/pjpeg", "image/png", "image/heic", "image/heif", "video/mp4", "video/quicktime", "video/hevc", "image/gif", "image/webp"]}

utils/pgclass.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, String, TIMESTAMP, func, BIGINT
+from sqlalchemy import Column, Integer, String, TIMESTAMP, func, BIGINT, LargeBinary
 from sqlalchemy.ext.declarative import declarative_base
 
 Base = declarative_base()
@@ -21,10 +21,22 @@ class SQLarticle(Base):
 class SQLlog(Base):
     __tablename__ = 'logs'
     
-    id = Column(Integer, primary_key=True)
+    id = Column(BIGINT, primary_key=True)
     created_at = Column(TIMESTAMP(timezone=True), server_default=func.now())
     message = Column(String)
     source = Column(String)
 
     def __repr__(self):
-        return f"<article(id={self.id}, created_at={self.created_at}, message={self.message}, source={self.source})>"
+        return f"<log(id={self.id}, created_at={self.created_at}, message={self.message}, source={self.source})>"
+    
+class SQLfile(Base):
+    __tablename__ = 'files'
+    
+    id = Column(BIGINT, primary_key=True)
+    created_at = Column(TIMESTAMP(timezone=True), server_default=func.now())
+    type = Column(String)
+    reference = Column(String)
+    binary = Column(LargeBinary)
+
+    def __repr__(self):
+        return f"<file(id={self.id}, created_at={self.created_at}, type={self.type}, binary={self.binary}, reference={self.reference})>"