python Google Cloud Python - 示例

Posted 2021-05-09

'''Example usage of datastore population, access, and deletion from
 A combnation of examples found under the datastore folder found via Google on
  Github: https://github.com/GoogleCloudPlatform/google-cloud-python/ 
'''

    
'''--------------------------------------------------------------------------'''
## Simple example of client Retrieval

from google.cloud import datastore

client = datastore.Client()
product_key = client.key('Product', 123)
print(client.get(product_key))

'''--------------------------------------------------------------------------'''


from __future__ import print_function

import os

import six

from google.cloud import datastore


ANCESTOR = ('Book', 'GoT')

RICKARD = ANCESTOR + ('Character', 'Rickard')

EDDARD = RICKARD + ('Character', 'Eddard')

KEY_PATHS = (
    RICKARD,
    EDDARD,
    ANCESTOR + ('Character', 'Catelyn'),
    EDDARD + ('Character', 'Arya'),
    EDDARD + ('Character', 'Sansa'),
    EDDARD + ('Character', 'Robb'),
    EDDARD + ('Character', 'Bran'),
    EDDARD + ('Character', 'Jon Snow'),
)


CHARACTERS = (
    {
        'name': u'Rickard',
        'family': u'Stark',
        'appearances': 0,
        'alive': False,
    }, {
        'name': u'Eddard',
        'family': u'Stark',
        'appearances': 9,
        'alive': False,
    }, {
        'name': u'Catelyn',
        'family': [u'Stark', u'Tully'],
        'appearances': 26,
        'alive': False,
    }, {
        'name': u'Arya',
        'family': u'Stark',
        'appearances': 33,
        'alive': True,
    }, {
        'name': u'Sansa',
        'family': u'Stark',
        'appearances': 31,
        'alive': True,
    }, {
        'name': u'Robb',
        'family': u'Stark',
        'appearances': 22,
        'alive': False,
    }, {
        'name': u'Bran',
        'family': u'Stark',
        'appearances': 25,
        'alive': True,
    }, {
        'name': u'Jon Snow',
        'family': u'Stark',
        'appearances': 32,
        'alive': True,
    },
)


FETCH_MAX = 20
ALL_KINDS = (
    'Character',
    'Company',
    'Kind',
    'Person',
    'Post',
)
TRANSACTION_MAX_GROUPS = 5


def print_func(message):
    if os.getenv('GOOGLE_CLOUD_NO_PRINT') != 'true':
        print(message)


def add_characters(client=None):
    if client is None:
        # Get a client that uses the test dataset.
        client = datastore.Client()
    with client.transaction() as xact:
        for key_path, character in six.moves.zip(KEY_PATHS, CHARACTERS):
            if key_path[-1] != character['name']:
                raise ValueError(('Character and key don\'t agree',
                                  key_path, character))
            entity = datastore.Entity(key=client.key(*key_path))
            entity.update(character)
            xact.put(entity)
            print_func('Adding Character %s %s' % (character['name'],
                                                   character['family']))


if __name__ == '__main__':
    add_characters()

# End Populating    


# Begin Clearing


def print_func(message):
    if os.getenv('GOOGLE_CLOUD_NO_PRINT') != 'true':
        print(message)


def fetch_keys(kind, client, fetch_max=FETCH_MAX, query=None, cursor=None):
    if query is None:
        query = client.query(kind=kind)
        query.keys_only()

    iterator = query.fetch(limit=fetch_max, start_cursor=cursor)
    page = six.next(iterator.pages)
    return query, list(page), iterator.next_page_token


def get_ancestors(entities):
    # NOTE: A key will always have at least one path element.
    key_roots = [entity.key.flat_path[:2] for entity in entities]
    # Return the unique roots.
    return list(set(key_roots))


def remove_kind(kind, client):
    results = []

    query, curr_results, cursor = fetch_keys(kind, client)
    results.extend(curr_results)
    while curr_results:
        query, curr_results, cursor = fetch_keys(
            kind, client, query=query, cursor=cursor)
        results.extend(curr_results)

    if not results:
        return

    delete_outside_transaction = False
    with client.transaction():
        # Now that we have all results, we seek to delete.
        print_func('Deleting keys:')
        print_func(results)

        ancestors = get_ancestors(results)
        if len(ancestors) > TRANSACTION_MAX_GROUPS:
            delete_outside_transaction = True
        else:
            client.delete_multi([result.key for result in results])

    if delete_outside_transaction:
        client.delete_multi([result.key for result in results])


def remove_all_entities(client=None):
    if client is None:
        # Get a client that uses the test dataset.
        client = datastore.Client()
    for kind in ALL_KINDS:
        remove_kind(kind, client)


if __name__ == '__main__':
    print_func('This command will remove all entities for '
               'the following kinds:')
    print_func('\n'.join('- ' + val for val in ALL_KINDS))
    response = six.moves.input('Is this OK [y/n]? ')
    if response.lower() == 'y':
        remove_all_entities()
    else:
        print_func('Doing nothing.')
        
        
# end Clearing

import logging
import threading
import time
import weakref


logger = logging.getLogger(__name__)


class ResourcePool(object):
    """Manage a pool of resourcess.
    There's no limit on the number of resourcess a pool can keep track of,
    but a warning is logged if there are more than ``pool.size`` active
    resources, and a critical problem if more than twice ``pool.size``.
    New resourcess are registered via add().  This will log a message if
    "too many" resourcess are active.
    When a resource is explicitly closed, return it via ``pool.check_in()``.
    That adds the resource to a stack of resources available for
    reuse, and throws away the oldest stack entries if the stack is too
    large.  ``pool.check_out()`` pops this stack.
    When a resource is obtained via ``pool.check_out()``, the pool holds only a
    weak reference to it thereafter.  It's not necessary to inform the pool
    if the resource goes away.  A resource handed out by ``pool.check_out()``
    counts against ``pool.size`` only so long as it exists, and provided it
    isn't returned via ``pool.check_in()``.
    
    We retain weak references to "checked out" resources  to allow debugging
    / monitoring.
    """

    def __init__(self, size=4, timeout=1<<31, logger=logger):

        self._lock = threading.RLock()
        self._logger = logger
        self._size = size
        self._timeout = timeout  # seconds

        # A weak mapping, id(resource) -> resource.
        self._all = weakref.WeakValueDictionary()

        # A stack of resources available to check out.
        self._available = []

    def __enter__(self):  # pragma: no cover
        return self._lock.__enter__()

    def __exit__(self, etype, err, tb):  # pragma: no cover
        return self._lock.__exit__(etype, err, tb)

    def __iter__(self):
        with self._lock:
            return iter(list(self._all.values()))

    @property
    def size(self):
        """Expected maximum # of live resources.
        """
        return self._size

    @size.setter
    def size(self, size):
        """Change the expected maximum # of live resources.
        If the new size is smaller than the current value, this may discard
        the oldest available resources.
        """
        with self._lock:
            self._size = size
            self._shrink(size)

    @property
    def timeout(self):
        """Max # of seconds to keep a resource in the pool.
        """
        return self._timeout

    @timeout.setter
    def timeout(self, timeout):
        """Change the max # of seconds to keep a resource in the pool.
        If the new timeout is smaller than the old value, this may discard
        the oldest available resources.
        """
        with self._lock:
            old, self._timeout = self._timeout, timeout
            if timeout < old:
                self._shrink(self.size)

    @property
    def available(self):
        """Return a set of the available connections.
        """
        with self._lock:
            return set([resource for timestamp, resource in self._available])

    def add(self, resource, checked_out=False):
        """Add a new resource to the pool.
        Raise ValueError if ``resource`` is already in the pool.
        If ``checked_out`` is False, push ``resource`` onto the available
        stack even if we're over the pool size limit (but shrink the pool
        if needed, potentially discarding older resources).
        If ``checked_out`` is True, do *not* push the resource onto
        the available stack:  the caller will presumably release it later
        via a call to ``check_in``.
        """
        with self._lock:
            if id(resource) in self._all:
                raise ValueError("Resource already in the pool")
            self._all[id(resource)] = resource
            if not checked_out:
                self._shrink(self.size - 1)
                self._append(resource)
            n = len(self._all)
            if n > self.size:
                reporter = self._logger.warning
                if n > 2 * self.size:
                    reporter = self._logger.critical
                reporter("Pool has %s resources with a size of %s",
                        n, self.size)

    def check_in(self, resource):
        """Release a checked-out resource back to the pool.
        Push ``resource`` onto the stack of available resources.
        
        May discard older available resources.
        """
        with self._lock:
            if id(resource) not in self._all:
                raise ValueError("Unknown resource:  use 'add()'")
            if resource in self.available:
                raise ValueError("Resource already checked in")
            self._shrink(self.size - 1)
            self._append(resource)

    def check_out(self):
        """Pop an available resource and return it.
        Return None if none are available.  In that case, the caller might
        create a new resource and register it via ``add()``, passing the
        ``checked_out`` flag to retain use of the resource..
        """
        with self._lock:
            if self._available:
                return self._available.pop()[1]

    def _append(self, resource):
        """Push a timestamped resource onto the stack available for checkout.
        Assumes ``self._lock`` is already acquired.
        """
        self._available.append((time.time(), resource))

    def _shrink(self, target):
        """Discard oldest available resources to meet the given target size.
        Assumes ``self._lock`` is already acquired.
        """
        threshhold = time.time() - self.timeout

        available = self._available
        while (len(available) > target or
              available and available[0][0] < threshhold):
            resource = available.pop(0)[1]
            del self._all[id(resource)]