From 80aa5e3a7de20976be63c889671824e625dd71eb Mon Sep 17 00:00:00 2001 From: XTao Date: Sun, 24 Jan 2016 20:12:41 +0800 Subject: [PATCH 1/9] Remove beansdb --- code.env | 1 - docker-compose.yml | 6 -- scripts/common.sh | 10 --- scripts/ubuntu.sh | 3 - tools/issue_bug.py | 27 ------- vilya/config.py | 6 -- vilya/libs/dbclient.py | 138 ------------------------------------ vilya/libs/redis2beansdb.py | 18 +++++ vilya/libs/store.py | 13 ++-- 9 files changed, 27 insertions(+), 195 deletions(-) delete mode 100755 tools/issue_bug.py delete mode 100644 vilya/libs/dbclient.py create mode 100644 vilya/libs/redis2beansdb.py diff --git a/code.env b/code.env index 231a92b..b3d982e 100644 --- a/code.env +++ b/code.env @@ -11,5 +11,4 @@ DOUBAN_CODE_DOUBANDB_SERVERS=memcached:11311 DOUBAN_CODE_DOUBANDB_PROXIES= DOUBAN_CODE_MYSQL_STORE={"farms": {"code_farm": {"master": "mysql:3306:valentine:douban_code:my-code-passwd", "tables": ["*"]}}} DOUBAN_CODE_REDIS_URI=redis://redis:6379/0 -DOUBAN_CODE_BEANSDB_HOSTS=beansdb:7900 DOUBAN_CODE_DOMAIN=http://192.168.99.100:8200 diff --git a/docker-compose.yml b/docker-compose.yml index 1dbaef8..4d0760d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,6 @@ web: links: - "mysql" - "memcached" - - "beansdb" - "redis" webpack: @@ -45,8 +44,3 @@ redis: image: redis:3.0 ports: - "6379:6379" - -beansdb: - image: mozillazg/beansdb:0.7.1.4 - ports: - - "7900:7900" diff --git a/scripts/common.sh b/scripts/common.sh index 14d580e..0212006 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -41,13 +41,3 @@ install_code() { start_app() { test "$CODE_ENV" != "unset" && gunicorn -w 2 -b 127.0.0.1:8000 app:app # web & git http daemon } - -install_beansdb() { - git clone https://github.com/douban/beansdb - cd beansdb - ./autogen.sh - ./configure && make && sudo make install - cd - - sudo mkdir /data/beansdb -p - beansdb -p 7900 -H /data/beansdb -T 0 -d -} diff --git a/scripts/ubuntu.sh b/scripts/ubuntu.sh index 4713b79..b176aba 100755 --- a/scripts/ubuntu.sh +++ b/scripts/ubuntu.sh @@ -14,9 +14,6 @@ echo "Setup memcached port to 11311..." sudo sed -i "s/11211/11311/g" /etc/memcached.conf sudo /etc/init.d/memcached restart -echo "Install beansdb..." -install_beansdb - echo "Install code..." install_code diff --git a/tools/issue_bug.py b/tools/issue_bug.py deleted file mode 100755 index c496d37..0000000 --- a/tools/issue_bug.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- - -from vilya.libs.store import store, bdb - -BDB_ISSUE_DESCRIPTION_KEY = 'issue_description:%s' - - -def main(): - ds = [] - rs = store.execute("select id, project_id, issue_id " - "from project_issues") - for r in rs: - id = r[0] - project_id = r[1] - issue_id = r[2] - description = bdb.get(BDB_ISSUE_DESCRIPTION_KEY % id) - bdb.delete(BDB_ISSUE_DESCRIPTION_KEY % id) - ds.append([issue_id, description]) - - for d in ds: - issue_id = d[0] - description = d[1] - bdb.set(BDB_ISSUE_DESCRIPTION_KEY % issue_id, description) - - -if __name__ == "__main__": - main() diff --git a/vilya/config.py b/vilya/config.py index e49fb01..d747078 100644 --- a/vilya/config.py +++ b/vilya/config.py @@ -57,12 +57,6 @@ REDIS_URI = os.environ.get('DOUBAN_CODE_REDIS_URI', 'redis://localhost:6379/0') -_beansdb_hosts = os.environ.get( - 'DOUBAN_CODE_BEANSDB_HOSTS', - 'localhost:7901,localhost:7902,localhost:7903' -).split(',') -BEANSDBCFG = {host: range(16) for host in _beansdb_hosts} - DOMAIN = os.environ.get('DOUBAN_CODE_DOMAIN', 'http://127.0.0.1:8200') IRC_SERVER = 'irc.intra.douban.com' IRC_PORT = 12345 diff --git a/vilya/libs/dbclient.py b/vilya/libs/dbclient.py deleted file mode 100644 index 7039ea2..0000000 --- a/vilya/libs/dbclient.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/python - -from random import shuffle - -from libmc import Client - - -def fnv1a(s): - prime = 0x01000193 - h = 0x811c9dc5 - for c in s: - h ^= ord(c) - h = (h * prime) & 0xffffffff - return h - - -class MCStore(object): - def __init__(self, server): - self.server = server - self.mc = Client([server], do_split=0) - - def __del__(self): - self.close() - - def __repr__(self): - return '' % repr(self.server) - - def __str__(self): - return self.server - - def set(self, key, data, rev=0): - return bool(self.mc.set(key, data, rev)) - - def set_raw(self, key, data, rev=0, flag=0): # interface for sync data - if rev < 0: - raise str(rev) - return self.mc.set_raw(key, data, rev, flag) - - def get(self, key): - try: - r = self.mc.get(key) - if r is None and self.mc.get_last_error() != 0: - raise IOError(self.mc.get_last_error()) - return r - except ValueError: - self.mc.delete(key) - - def get_raw(self, key): - r, flag = self.mc.get_raw(key) - if r is None and self.mc.get_last_error() != 0: - raise IOError( - self.mc.get_last_error(), self.mc.get_last_strerror()) - return r, flag - - def get_multi(self, keys): - return self.mc.get_multi(keys) - - def delete(self, key): - return bool(self.mc.delete(key)) - - def close(self): - if self.mc is not None: - self.mc = None - - -class WriteFailedError(Exception): - def __init__(self, key): - self.key = key - - def __repr__(self): - return 'write %s failed' % self.key - - -class Beansdb(object): - hash_space = 1 << 32 - cached = True - - def __init__(self, servers, buckets_count=16, N=3, W=1, R=1): - self.buckets_count = buckets_count - self.bucket_size = self.hash_space / buckets_count - self.servers = {} - self.server_buckets = {} - self.buckets = [[] for i in range(buckets_count)] - for s, bs in servers.items(): - server = MCStore(s) - self.servers[s] = server - self.server_buckets[s] = bs - for b in bs: - self.buckets[b].append(server) - for b in range(self.buckets_count): - self.buckets[b].sort(key=lambda x: fnv1a("%d:%s:%d" % (b, x, b))) - self.N = N - self.W = W - self.R = R - - def print_buckets(self): - for i, ss in enumerate(self.buckets): - print i, ','.join(str(s) for s in ss) - for s, bs in self.server_buckets.items(): - print s, len(bs) - - def _get_servers(self, key): - hash = fnv1a(key) - b = hash / self.bucket_size - return self.buckets[b] - - def get(self, key, default=None): - ss = self._get_servers(key) - for i, s in enumerate(ss): - r = s.get(key) - if r is not None: - # self heal - for k in range(i): - ss[k].set(key, r) - return r - return default - - def get_multi(self, keys): - rs = {} - ss = self.servers.values() - shuffle(ss) - for s in ss: - rs.update(s.get_multi( - [k for k in keys if k not in rs and s in self._get_servers(k)] - )) - return rs - - def delete(self, key): - rs = [s.delete(key) for s in self._get_servers(key)] - return rs.count(True) >= self.W - - def set(self, key, value): - rs = [s.set(key, value) for s in self._get_servers(key)] - if not rs.count(True) >= self.W: - # try to get, it will return False when set same content into db - if self.get(key) != value: - raise WriteFailedError(key) - return True diff --git a/vilya/libs/redis2beansdb.py b/vilya/libs/redis2beansdb.py new file mode 100644 index 0000000..4a644a3 --- /dev/null +++ b/vilya/libs/redis2beansdb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +import redis +from vilya.config import REDIS_URI + + +class Beansdb(object): + + def __init__(self): + self.redis = redis.from_url(REDIS_URI) + + def get(self, key): + return self.redis.get(key) + + def set(self, key, value): + return self.redis.set(key, value) + + def delete(self, key): + return self.redis.delete(key) diff --git a/vilya/libs/store.py b/vilya/libs/store.py index cb28685..71963f7 100644 --- a/vilya/libs/store.py +++ b/vilya/libs/store.py @@ -7,9 +7,12 @@ from ORZ import setup, OrzField, OrzBase # noqa from douban.mc import create_decorators from douban.sqlstore import store_from_config -from .dbclient import Beansdb +from redis2beansdb import Beansdb from vilya.config import ( - MEMCACHED_HOSTS, MEMCACHED_CONFIG, MYSQL_STORE, BEANSDBCFG) + MEMCACHED_HOSTS, + MEMCACHED_CONFIG, + MYSQL_STORE, +) logging.getLogger().setLevel(logging.DEBUG) @@ -52,7 +55,7 @@ def get_mc(): def get_db(): - return Beansdb(BEANSDBCFG, 16) + return Beansdb() def stub_cache(*args, **kws): @@ -87,11 +90,13 @@ def reset_mc(deep=False): getattr(mc.mc, 'clear', lambda: True)() +# TODO(xutao) delete reset_beansdb? def reset_beansdb(): for db in _clients.itervalues(): db.clear_cache() +# FIXME(xutao) rename to clear_mc_for_test? def clear_beansdb_for_test(): for db in _clients.itervalues(): db.clear_cache() @@ -100,7 +105,7 @@ def clear_beansdb_for_test(): if clear: clear() - +# FIXME(xutao) I can't find any code about put beansdb client to `_clients`, and beansdb shouldn't be clear after request def clear_local_cache(): reset_mc() reset_beansdb() From 1d47276be327b15e53383a0b60d9d99e9ae66472 Mon Sep 17 00:00:00 2001 From: Prayag Verma Date: Tue, 26 Jan 2016 14:34:39 +0530 Subject: [PATCH 2/9] Update license year range to 2016 --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 7c79be9..fc052a9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2013-2014, Douban Inc. +Copyright (c) 2013-2016, Douban Inc. All rights reserved. Redistribution and use in source and binary forms, with or without From 0f7a3e6c05a1d29c07c3e6b520073add8b09faa8 Mon Sep 17 00:00:00 2001 From: XTao Date: Sat, 6 Feb 2016 23:57:12 +0800 Subject: [PATCH 3/9] Remove switch --- dockerfiles/python/requirements.txt | 4 ++- vilya/libs/redis2beansdb.py | 7 +++-- vilya/models/project.py | 15 ---------- vilya/models/utils/switch.py | 38 ------------------------ vilya/templates/widgets/pagehead.html | 10 ------- vilya/templates/widgets/pagehead.mako | 10 ------- vilya/templates/widgets/pjax_source.html | 4 --- 7 files changed, 8 insertions(+), 80 deletions(-) delete mode 100644 vilya/models/utils/switch.py diff --git a/dockerfiles/python/requirements.txt b/dockerfiles/python/requirements.txt index 6406a76..b0cebe6 100644 --- a/dockerfiles/python/requirements.txt +++ b/dockerfiles/python/requirements.txt @@ -9,7 +9,6 @@ Werkzeug==0.9.4 celery==3.1.18 redis==2.10.3 blinker==1.4 -paramiko==1.15.2 PyYAML==3.11 Sphinx==1.3.1 PyRSS2Gen==1.1 @@ -33,3 +32,6 @@ docutils==0.12 -e git+https://github.com/douban/pygit2.git@c7b13237e9934b5620cf0c351ae7eaea7d14ffc3#egg=pygit2 -e git+https://github.com/douban/ellen.git@e092eddd933a4676c53f016d11782897807f873a#egg=ellen -e git+https://github.com/douban/libmc.git@06b27e8bf2ae7f75d9817ba917f705c471e45028#egg=libmc +kombu==3.0.33 +billiard==3.3.0.22 +paramiko==1.16.0 diff --git a/vilya/libs/redis2beansdb.py b/vilya/libs/redis2beansdb.py index 4a644a3..8ce506d 100644 --- a/vilya/libs/redis2beansdb.py +++ b/vilya/libs/redis2beansdb.py @@ -8,8 +8,11 @@ class Beansdb(object): def __init__(self): self.redis = redis.from_url(REDIS_URI) - def get(self, key): - return self.redis.get(key) + def get(self, key, default=None): + data = self.redis.get(key) + if not data: + return default + return data def set(self, key, value): return self.redis.set(key, value) diff --git a/vilya/models/project.py b/vilya/models/project.py index 784d779..ca73959 100644 --- a/vilya/models/project.py +++ b/vilya/models/project.py @@ -30,7 +30,6 @@ from vilya.models.lru_counter import ( ProjectOwnLRUCounter, ProjectWatchLRUCounter) from vilya.models.milestone import Milestone -from vilya.models.utils.switch import WhiteListSwitch from ellen.utils import JagareError from vilya.models.nproject import ProjectWatcher @@ -82,20 +81,6 @@ def __hash__(self): def __str__(self): return self.name - @property - def dashboard_enabled(self): - dashboard_enabled_switch = WhiteListSwitch( - 'DASHBOARD_ENABLED_PROJECTS') - enabled_projects = dashboard_enabled_switch.get() - return self.name in enabled_projects - - @property - def deploy_link_enabled(self): - deploy_link_enabled_switch = WhiteListSwitch( - 'DEPLOY_LINK_ENABLED_PROJECTS') - enabled_projects = deploy_link_enabled_switch.get() - return self.name in enabled_projects - @property def url(self): return '/%s/' % self.name diff --git a/vilya/models/utils/switch.py b/vilya/models/utils/switch.py deleted file mode 100644 index 536e11a..0000000 --- a/vilya/models/utils/switch.py +++ /dev/null @@ -1,38 +0,0 @@ -# coding: utf-8 - -from vilya.libs.store import bdb - - -class WhiteListSwitch(object): - - DBKEY_SWITCH_PREFIX = "SWITCH_" - - def __init__(self, name): - self.name = name - - @property - def dbkey(self): - return "%s%s" % (self.DBKEY_SWITCH_PREFIX, self.name) - - def get(self): - return bdb.get(self.dbkey, []) - - def set(self, white_list): - bdb.set(self.dbkey, white_list) - return bdb.get(self.dbkey, []) - - def add(self, one): - white_list = bdb.get(self.dbkey, []) - if one not in white_list: - white_list.append(one) - bdb.set(self.dbkey, white_list) - return bdb.get(self.dbkey, []) - return white_list - - def remove(self, one): - white_list = bdb.get(self.dbkey, []) - if one in white_list: - white_list.remove(one) - bdb.set(self.dbkey, white_list) - return bdb.get(self.dbkey, []) - return white_list diff --git a/vilya/templates/widgets/pagehead.html b/vilya/templates/widgets/pagehead.html index 03a3165..63e66d3 100644 --- a/vilya/templates/widgets/pagehead.html +++ b/vilya/templates/widgets/pagehead.html @@ -25,11 +25,6 @@

% endif - % if project.dashboard_enabled: - - - - % endif

% if user: @@ -63,11 +58,6 @@

  • Code
    • - % if project.dashboard_enabled: -
  • - Dashboard -
    • - % endif % if project.conf['docs']: % for doc, vals in sorted(project.conf['docs'].items(), key=lambda x: x[1].get('sort', x[0])):
  • ${vals.get('name', doc)}
    • diff --git a/vilya/templates/widgets/pagehead.mako b/vilya/templates/widgets/pagehead.mako index 98f0754..36c2e1a 100644 --- a/vilya/templates/widgets/pagehead.mako +++ b/vilya/templates/widgets/pagehead.mako @@ -59,11 +59,6 @@ % endif - % if project.dashboard_enabled: - - - - % endif

% if user: @@ -121,11 +116,6 @@