bsc-s2 / pykit Goto Github PK

pykit.http.Client.request(**argkv, body=None)

#!/usr/bin/env python2
# coding: utf-8

import os
import pkgutil

def submodules(root_mod):

    mod_path = root_mod.__file__

    fn = os.path.basename( mod_path )
    pathname = os.path.dirname( mod_path )

    if fn not in ("__init__.py", "__init__.pyc"):
        return None

    rst = {}

    for imp, name, _ in pkgutil.iter_modules( [ pathname ] ):
        loader = imp.find_module( name )
        mod = loader.load_module( root_mod.__name__ + "." + name )
        rst[ name ] = mod

    return rst

def submodule_tree(root):

    r = submodules( root )
    if r is None:
        return None

    for name, mod in r.items():
        children = submodule_tree( mod )
        r[ name ] = { "mod": mod, "children": children }
    return r

# Return a dictionary with non-leaf node representing module path and leaf node
# referencing to module.
# Thus non-leaf module would not be returned.
# {
#   "s2role": {
#       "Front": {
#           "python": { "nodeAPI": <module> },
#        }
#    }
#}
def submodule_leaf_tree(root):

    r = submodules( root )
    if r is None:
        return None

    for name, mod in r.items():
        children = submodule_leaf_tree( mod )
        if children is None:
            r[ name ] = mod
        else:
            r[ name ] = children

    return r

def rmtree(path, ignore_errors=False, onerror=None):
    """Recursively delete a directory tree.

    If ignore_errors is set, errors are ignored; otherwise, if onerror
    is set, it is called to handle the error with arguments (func,
    path, exc_info) where func is os.listdir, os.remove, or os.rmdir;
    path is the argument to that function that caused it to fail; and
    exc_info is a tuple returned by sys.exc_info().  If ignore_errors
    is false and onerror is None, an exception is raised.

    """
    if ignore_errors:
        def onerror(*args):
            pass
    elif onerror is None:
        def onerror(*args):
            raise
    try:
        if os.path.islink(path):
            # symlinks to directories are forbidden, see bug #1669
            raise OSError("Cannot call rmtree on a symbolic link")
    except OSError:
        onerror(os.path.islink, path, sys.exc_info())
        # can't continue even if onerror hook returns
        return
    names = []
    try:
        names = os.listdir(path)
    except os.error, err:
        onerror(os.listdir, path, sys.exc_info())
    for name in names:
        fullname = os.path.join(path, name)
        try:
            mode = os.lstat(fullname).st_mode
        except os.error:
            mode = 0
        if stat.S_ISDIR(mode):
            rmtree(fullname, ignore_errors, onerror)
        else:
            try:
                os.remove(fullname)
            except os.error, err:
                onerror(os.remove, fullname, sys.exc_info())
    try:
        os.rmdir(path)
    except os.error:
        onerror(os.rmdir, path, sys.exc_info())

convert({ 1: "x", 2: "y"}, [1,2], [2])
# 使用foo[1]的值作为第一级的key, 使用foo[2]作为第2级的key;
# 输出:
{ "x": { "y": "y" } }

convert({ 1: "x", 2: { 3: "z", 4: "w" }}, [(2,3), 1], [2])
# 使用 foo[2][3] 的值作为第一级的key. 使用 foo[1] 的值作为第2级的key;
# 输出:
{ "z": { "x": { 3: "z", 4: "w" }}}

{
    "db": which database to sharding. a stirng.
    "table":  which table to sharding. a string.
    "conn": {"host", "port", "user", "passwd"},
    "shard_fields": those fileds to sharding by, a list or tuple.
    "first_shard": first shard values in the table, a list or tuple.
    "number_per_shard": how many rows of data a shard can contain and its tolerance. a list or tuple like (number, number).
    "sharding_generator": a function that valid sharding format use a `shard` ad argument.
}

{
    "shard":  sharding info, [(), (), ...],
    "number": numbers of rows in every shard, [number, number, ...],
    "all": number of all the rows in every shard. a number.
}

def argssplit( argstr ):

    args = []

    argstr = argstr.strip()

    m = len(argstr)
    i = 0

    while( i < m ):

        j = i

        if argstr[j] in ('"',"'"):

            i += 1
            while( i < m and argstr[i] != argstr[j] ):
                i += 1

            args.append(argstr[j+1:i])
            i += 1

        else :

            while( i < m and argstr[i] != ' ' ):
                i += 1

            args.append(argstr[j:i])

        while( i < m and argstr[i] == ' ' ):
            i += 1

    return args

break_line('foo bar bar.', 9)
# foo bar
# bar

mysqlutil.sql_dump_between_shards(shard_fileds, dbinfo, tablename, sql_path, bin_path, start, end=None)

# default
sql_dump_between_shards(
    ['bucket_id', 'scope', 'key'],
    {
        'host':'127.0.0.1',
        'user':'root',
        'password':'pw',
        'port':3600,
        'db':'mysql'
     },
     'key',
     ['tmp', 'key.sql'],
     ['mysqldump'],
     (100000000, 'a', 'key_foo'),
     (200000000, 'a', 'key_bar')
)

# " mysqldump --host=127.0.0.1 --port=3600 --user=user --password='pw' mysql key -w 
# (`bucket_id`=100000000 AND `scope`='a' AND `key`>='key_foo') OR
# (`bucket_id`=100000000 AND `scope`>'a') OR 
# (`bucket_id`>100000000 AND `bucket_id`<200000000) OR
# (`bucket_id`=200000000 AND `scope`<'a') OR
# (`bucket_id`=200000000 AND `scope`='a' AND `key`<'key_bar')
# > tmp/key.sql"

# demo.py

from pykit import shell

arguments = {
    'echo_repr': (
                lambda *x: sys.stdout.write(repr(x)),
                {'x': {'nargs': '+', 'help': 'just an input message'}},
             ),

    'foo': {
        'bar': sys.stdout.write('bar'),

        'bob': {
            'plus': (
                   lambda x, y: sys.stdout.write(x + y),
                   {'x': {'nargs': 1, 'type'=int, help='an int is needed'}},
                   {'y': {'nargs': 1, 'type'=int, help='an int is neeedd'}},
                ),
        },
    },

    '__add_help__': {
        ('echo_repr',)          : 'output what is input.',
        ('foo', 'bar',)         : 'print a "bar".',
        ('foo', 'bob', 'plus')  : 'do addition operation with 2 numbers.',
    },

    '__description__': 'this is an example command.',
}

shell.command(**arguments)

$ python demo.py -h
---------------------------
usage: this is an example command. [-h] {echo_repr, foo bar, foo bob plus} ...

positional arguments:
  {echo_repr, foo bar, foo bob plus} commands...
    echo_repr          output what is input.
    foo bar            print a "bar".
    foo bob plus       do addition operation with 2 numbers.

optional arguments:
    -h, --help           show this help message and exit


$ python demo.py foo bob plus -h
--------------------------
usage: this is an example command. foo bob plus [-h] x y
positional arguments:
    x   an int is needed
    y   an int is needed

optional arguments:
    -h, --help           show this help message and exit

lock = threading.RLock()

def atomic_write_file( fn, fcont ):

    with lock:
        tmpfn = fn + "._tmp_." + str(os.getpid()) + str(hash(fcont))
        fsutil.write_file(tmpfn, fcont, uid=conf.s2uid, gid=conf.s2gid)

        os.rename( tmpfn, fn )

a = RangeSet([[1,7]])
b = RangeSet([[2,4]])

a.substract(b) # [[1,2], [4,7]]
a.intersect(b)  # [[2,4]]
a.union(b)       # [[1,7]]
RangeSet([[1,2],[5,7]], ).union([[3,6]]) # [[1,2], [3,7]]

def makedirs(*paths, **kwargs):
    mode = kwargs.get('mode', 0755)
    uid = kwargs.get('uid')
    gid = kwargs.get('uid')

def write_file(path, fcont, uid=None, gid=None, atomic=False):
--
 
if not atomic:
return _write_file(path, fcont, uid, gid)
 
tmp_path = '{path}._tmp_.{pid}_{timestamp}'.format(
path=path,
pid=os.getpid(),
timestamp=timeutil.ns(),
)
_write_file(tmp_path, fcont, uid, gid)
 
os.rename(tmp_path, path)

mysqlutil.sql_condition_between_shards(index_fields, shard, next_shard)

# default
sql_condition_between_shards([bucket_id, scope, key], (100000000, 'a', 'key_foo'), (200000000, 'a', 'key_bar'))
# ["`bucket_id`=100000000 AND `scope`='a' AND `key`>='key_foo'",
# "`bucket_id`=100000000 AND `scope`>'a'",
# "`bucket_id`>100000000 AND `bucket_id`<200000000",
# "`bucket_id`=200000000 AND `scope`<'a'",
# "`bucket_id`=200000000 AND `scope`='a' AND `key`<'key_bar'",]

# first index_field equals
sql_condition_between_shards([bucket_id, scope, key], (100000000, 'a', 'key_foo'), (100000000, 'b', 'key_bar'))
# ["`bucket_id`=100000000 AND `scope`='a' AND `key` >= 'key_foo'",
# "`bucket_id`=100000000 AND `scope`>'a' AND `scope`<'b'",
# "`bucket_id`=100000000 AND `scope` = 'b' AND `key` < 'key_bar'",]

# the first two index_field equals
sql_condition_between_shards([bucket_id, scope, key], (100000000, 'a', 'key_foo'), (100000000, 'a', 'key_bar'))
# ["`bucket_id`=100000000 AND `scope`='a' AND `key` >= 'key_foo' AND `key` < 'key_bar'"]