diff --git a/.gitignore b/.gitignore index e38c008..72c0803 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,11 @@ __pycache__/ *.py[cod] *$py.class + test*.py -db/ +db/*.db +db/versions/* + filecache/ # C extensions *.so diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..3d9dac6 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,84 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = db + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to db/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat db/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 +# sqlalchemy.url = driver://user:pass@localhost/dbname +sqlalchemy.url = sqlite:///db/userlist.db + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks=black +# black.type=console_scripts +# black.entrypoint=black +# black.options=-l 79 + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/core/callback.py b/core/callback.py index aa8e6e5..9523161 100644 --- a/core/callback.py +++ b/core/callback.py @@ -10,9 +10,9 @@ from threading import Thread from fbchat import ThreadType, Message, \ ShareAttachment, FileAttachment, AudioAttachment, VideoAttachment, ImageAttachment, Sticker, LocationAttachment +from db import models from lib import common from lib.facebook import FacebookClient -from lib.sqlhelper import UserList, Status from munch import Munch log = logging.getLogger(__name__) @@ -27,19 +27,19 @@ class CallBack(): print('【%s】' % type_, code, msg) def onLoggingIn(self, email, password, cookie, user_agent=None): - user_obj = UserList.get(email=email) + user_obj = models.UserList.get(email=email) if not user_obj: - user_obj = UserList.insert(email=email, password=password, status=Status.LOGGINE, cookie=cookie, + user_obj = models.UserList.insert(email=email, password=password, status=common.Status.LOGGINE, cookie=cookie, user_agent=user_agent) else: - user_obj.set(status=Status.LOGGINE) + user_obj.set(status=common.Status.LOGGINE) return user_obj def onLoggedIn(self, client: FacebookClient): client.user_obj.set( fbid=client.uid, - status=Status.ONLINE, + status=common.Status.ONLINE, cookie=client.get_cookie(), user_agent=client.get_user_agent() ) @@ -55,8 +55,8 @@ class CallBack(): def onLoggingError(self, email, reason, taskid=0): - u = UserList.get(email=email) - if u: u.set(status=Status.FAILED) + u = models.UserList.get(email=email) + if u: u.set(status=common.Status.FAILED) client = Munch(email=email) self._notify_( @@ -66,7 +66,7 @@ class CallBack(): ) def onLogout(self, client): - client.user_obj.set(status=Status.OFFLINE) + client.user_obj.set(status=common.Status.OFFLINE) self._notify_( type_="account", diff --git a/core/monitor.py b/core/monitor.py index 452de1f..be704f4 100644 --- a/core/monitor.py +++ b/core/monitor.py @@ -17,10 +17,10 @@ from munch import Munch from conf import settings from core import callback, command -from lib import control_server +from db import models +from lib import control_server, common from lib.common import TaskStatus from lib.facebook import FacebookClient -from lib.sqlhelper import UserList, Status, Config from utils import parameter log = logging.getLogger(__name__) @@ -35,9 +35,9 @@ class Monitor(callback.CallBack): super().__init__() self._socket = self._temp_socket = None self._listenlist = dict() - self._imei = Config.get('imei', lambda: uuid.uuid1().hex) - self._name = Config.get('name', control_server.get_init_name) - self._host = Config.get('host', lambda: settings.get_server()['url']) + self._imei = models.Config.get('imei', lambda: uuid.uuid1().hex) + self._name = models.Config.get('name', control_server.get_init_name) + self._host = models.Config.get('host', lambda: settings.get_server()['url']) self.version = None self.executor = ThreadPoolExecutor(50, 'task_thread') self.init_config = {} @@ -45,13 +45,13 @@ class Monitor(callback.CallBack): def bind(self, socket): self._socket = socket - Config.set('host', self._socket.ws_url) + models.Config.set('host', self._socket.ws_url) def go_login(self): threading.Thread(target=self._auto_login, args=(), name='auto_login_thread').start() def _auto_login(self): - user_list = UserList.query(status=Status.ONLINE) + user_list = models.UserList.query(status=common.Status.ONLINE) for user in user_list: print("自动登录->", user) self.login(user.email, user.password, user.format_cookie(), user.user_agent) diff --git a/db/README b/db/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/db/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/db/env.py b/db/env.py new file mode 100644 index 0000000..07f537d --- /dev/null +++ b/db/env.py @@ -0,0 +1,83 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata + +import os, sys + +sys.path.append(os.getcwd()) +from db.models import Base +target_metadata = Base.metadata + + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/db/models.py b/db/models.py new file mode 100644 index 0000000..88ab40f --- /dev/null +++ b/db/models.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os + +from sqlalchemy import Column, Integer, String, create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import SingletonThreadPool + +from conf.settings import user_db_path + +Base = declarative_base() +metadata = Base.metadata + +engine = create_engine('sqlite:///{}'.format(os.path.join(user_db_path, "userlist.db")), + poolclass=SingletonThreadPool, + connect_args={'check_same_thread': False}) + +Session = sessionmaker(bind=engine) +session = Session() + + +class Config(Base): + __tablename__ = 'config' + id = Column(Integer, primary_key=True, autoincrement=True) + key = Column(String(50), index=True, nullable=False, unique=True) + value = Column(String(256)) + + @staticmethod + def get(key, value_func=None): + conf = session.query(Config).filter_by(key=key).first() + if not conf: + if value_func: + conf = Config(key=key, value=value_func()) + session.add(conf) + session.commit() + return conf.value + return None + return conf.value + + @staticmethod + def set(key, value): + conf = session.query(Config).filter_by(key=key).first() + if not conf: + conf = Config(key=key, value=value) + session.add(conf) + session.commit() + return conf + else: + conf.value = value + session.commit() + return conf + + def __repr__(self): + return "Config(key={}, value={})".format(self.key, self.value) + + +class UserList(Base): + __tablename__ = 'users' + + id = Column(Integer, primary_key=True, autoincrement=True) + email = Column(String(50), index=True, nullable=False, unique=True) + password = Column(String(50), nullable=False) + cookie = Column(String(512)) + user_agent = Column(String(256)) + fbid = Column(String(20), index=True) + status = Column(Integer, default=0, nullable=False, index=True) + proxy = Column(String(256)) + + def __repr__(self): + return "User(id={}, email={}, password={}, cookie={}, fbid={}, status={})" \ + .format(self.id, self.email, self.password, len(self.cookie) if self.cookie else None, self.fbid, + self.status) + + def format_cookie(self): + if self.cookie: + return dict([tuple(sub.split("=")) for sub in self.cookie.split('; ')]) + else: + return {} + + def set(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + session.commit() + return self + + @staticmethod + def update(**kwargs): + unique = tuple(kwargs)[0] + if unique == 'email': + user_ = session.query(UserList).filter_by(email=kwargs.get(unique)).first() + elif unique == 'fbid': + user_ = session.query(UserList).filter_by(fbid=kwargs.get(unique)).first() + else: + raise BaseException("条件不对") + kwargs.pop(unique) + for k, v in kwargs.items(): + setattr(user_, k, v) + session.commit() + return user_ + + @staticmethod + def insert(**kwargs): + u = UserList(**kwargs) + session.add(u) + session.commit() + return u + + @staticmethod + def get(**kwargs): + unique = tuple(kwargs)[0] + if unique == 'email': + user_ = session.query(UserList).filter_by(email=kwargs.get(unique)).first() + elif unique == 'fbid': + user_ = session.query(UserList).filter_by(fbid=kwargs.get(unique)).first() + else: + raise BaseException("条件不对") + + return user_ + + @staticmethod + def remove(**kwargs): + unique = tuple(kwargs)[0] + if unique == 'email': + user_ = session.query(UserList).filter_by(email=kwargs.get(unique)).first() + elif unique == 'fbid': + user_ = session.query(UserList).filter_by(fbid=kwargs.get(unique)).first() + else: + raise BaseException("条件不对") + if user_: + session.delete(user_) + session.commit() + else: + raise BaseException("用户不存在") + + @staticmethod + def all() -> list: + users = session.query(UserList).all() + return users + + @staticmethod + def query(**kwargs) -> list: + users = session.query(UserList).filter_by(**kwargs).all() + return users diff --git a/db/script.py.mako b/db/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/db/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/lib/common.py b/lib/common.py index 60df5e0..baedfe7 100644 --- a/lib/common.py +++ b/lib/common.py @@ -156,6 +156,13 @@ class Exchange(Enum): return None +class Status(): + OFFLINE = 0 + ONLINE = 1 + LOGGINE = 2 + FAILED = 3 + + def todict(obj, include: list = None): keys = dir(obj) res = {} diff --git a/lib/sqlhelper.py b/lib/sqlhelper.py index 4f9a5d0..e69de29 100644 --- a/lib/sqlhelper.py +++ b/lib/sqlhelper.py @@ -1,166 +0,0 @@ -import asyncio -import os -import threading - -from sqlalchemy import Column, Integer, String -from sqlalchemy import create_engine -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker -from sqlalchemy.pool import SingletonThreadPool - -from conf.settings import user_db_path - -Base = declarative_base() - -engine = create_engine('sqlite:///{}'.format(os.path.join(user_db_path, "userlist.db")), - poolclass=SingletonThreadPool, - connect_args={'check_same_thread': False}) - -Session = sessionmaker(bind=engine) -session = Session() - - -class Status(): - OFFLINE = 0 - ONLINE = 1 - LOGGINE = 2 - FAILED = 3 - - -class Config(Base): - __tablename__ = 'config' - id = Column(Integer, primary_key=True, autoincrement=True) - key = Column(String(50), index=True, nullable=False, unique=True) - value = Column(String(256)) - - @staticmethod - def get(key, value_func=None): - conf = session.query(Config).filter_by(key=key).first() - if not conf: - if value_func: - conf = Config(key=key, value=value_func()) - session.add(conf) - session.commit() - return conf.value - return None - return conf.value - - @staticmethod - def set(key, value): - conf = session.query(Config).filter_by(key=key).first() - if not conf: - conf = Config(key=key, value=value) - session.add(conf) - session.commit() - return conf - else: - conf.value = value - session.commit() - return conf - - def __repr__(self): - return "Config(key={}, value={})".format(self.key, self.value) - - -class UserList(Base): - __tablename__ = 'users' - - id = Column(Integer, primary_key=True, autoincrement=True) - email = Column(String(50), index=True, nullable=False, unique=True) - password = Column(String(50), nullable=False) - cookie = Column(String(512)) - user_agent = Column(String(256)) - fbid = Column(String(20), index=True) - status = Column(Integer, default=0, nullable=False, index=True) - - # proxy = Column(String(256)) - - def __repr__(self): - return "User(id={}, email={}, password={}, cookie={}, fbid={}, status={})" \ - .format(self.id, self.email, self.password, len(self.cookie) if self.cookie else None, self.fbid, - self.status) - - def format_cookie(self): - if self.cookie: - return dict([tuple(sub.split("=")) for sub in self.cookie.split('; ')]) - else: - return {} - - def set(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - session.commit() - return self - - @staticmethod - def update(**kwargs): - unique = tuple(kwargs)[0] - if unique == 'email': - user_ = session.query(UserList).filter_by(email=kwargs.get(unique)).first() - elif unique == 'fbid': - user_ = session.query(UserList).filter_by(fbid=kwargs.get(unique)).first() - else: - raise BaseException("条件不对") - kwargs.pop(unique) - for k, v in kwargs.items(): - setattr(user_, k, v) - session.commit() - return user_ - - @staticmethod - def insert(**kwargs): - u = UserList(**kwargs) - session.add(u) - session.commit() - return u - - @staticmethod - def get(**kwargs): - unique = tuple(kwargs)[0] - if unique == 'email': - user_ = session.query(UserList).filter_by(email=kwargs.get(unique)).first() - elif unique == 'fbid': - user_ = session.query(UserList).filter_by(fbid=kwargs.get(unique)).first() - else: - raise BaseException("条件不对") - - return user_ - - @staticmethod - def remove(**kwargs): - unique = tuple(kwargs)[0] - if unique == 'email': - user_ = session.query(UserList).filter_by(email=kwargs.get(unique)).first() - elif unique == 'fbid': - user_ = session.query(UserList).filter_by(fbid=kwargs.get(unique)).first() - else: - raise BaseException("条件不对") - if user_: - session.delete(user_) - session.commit() - else: - raise BaseException("用户不存在") - - @staticmethod - def all() -> list: - users = session.query(UserList).all() - return users - - @staticmethod - def query(**kwargs) -> list: - users = session.query(UserList).filter_by(**kwargs).all() - return users - - -# Base.metadata.drop_all(engine) -Base.metadata.create_all(engine) - -if __name__ == '__main__': - print(UserList.all()) - # def tes(email): - # u = UserList.get(email=email) - # u.set(status=3) - # print(u) - # threading.Thread(target=tes, args=('HantzKlair97@mail.ru',)).start() - # threading.Thread(target=tes, args=('Evangelina.ve.59499@gmail.com',)).start() - pass diff --git a/requirements.txt b/requirements.txt index 5a089d0..4967402 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,5 @@ sqlalchemy==1.3.12 psutil==5.6.7 demjson==2.2.4 apscheduler==3.6.3 -pysocks==1.7.1 \ No newline at end of file +pysocks==1.7.1 +alembic==1.4.0 \ No newline at end of file