Merge branch 'gh_3481_file_type_detection' of github.com:eug3nix/lbry…

…-sdk into gh_3481_file_type_detection

lbry/dht/peer.py

@@ -8,10 +8,11 @@
 from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
 
 ALLOW_LOCALHOST = False
+CACHE_SIZE = 2048
 log = logging.getLogger(__name__)
 
 
-@lru_cache(1024)
+@lru_cache(CACHE_SIZE)
 def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional[str],
                        udp_port: typing.Optional[int] = None,
                        tcp_port: typing.Optional[int] = None,
@@ -29,13 +30,13 @@ def __init__(self, loop: asyncio.AbstractEventLoop):
         self._loop = loop
         self._rpc_failures: typing.Dict[
             typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
-        ] = {}
-        self._last_replied: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
-        self._last_sent: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
-        self._last_requested: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
-        self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = LRUCache(2048)
-        self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = LRUCache(2048)
-        self._node_tokens: typing.Dict[bytes, (float, bytes)] = LRUCache(2048)
+        ] = LRUCache(CACHE_SIZE)
+        self._last_replied: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE)
+        self._last_sent: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE)
+        self._last_requested: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE)
+        self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = LRUCache(CACHE_SIZE)
+        self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = LRUCache(CACHE_SIZE)
+        self._node_tokens: typing.Dict[bytes, (float, bytes)] = LRUCache(CACHE_SIZE)
 
     def reset(self):
         for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):

lbry/extras/daemon/daemon.py

@@ -2439,7 +2439,7 @@ async def jsonrpc_claim_search(self, **kwargs):
                          [--not_locations=<not_locations>...]
                          [--order_by=<order_by>...] [--no_totals] [--page=<page>] [--page_size=<page_size>]
                          [--wallet_id=<wallet_id>] [--include_purchase_receipt] [--include_is_my_output]
-                         [--remove_duplicates] [--has_source | --has_no_source]
+                         [--remove_duplicates] [--has_source | --has_no_source] [--sd_hash=<sd_hash>]
                          [--new_sdk_server=<new_sdk_server>]
 
         Options:
@@ -2537,6 +2537,8 @@ async def jsonrpc_claim_search(self, **kwargs):
             --remove_duplicates             : (bool) removes duplicated content from search by picking either the
                                                      original claim or the oldest matching repost
             --has_source                    : (bool) find claims containing a source field
+            --sd_hash=<sd_hash>             : (str)  find claims where the source stream descriptor hash matches
+                                                     (partially or completely) the given hexadecimal string
             --has_no_source                 : (bool) find claims not containing a source field
            --new_sdk_server=<new_sdk_server> : (str) URL of the new SDK server (EXPERIMENTAL)
 
@@ -2912,7 +2914,7 @@ async def jsonrpc_channel_sign(
             wallet, channel_account_id, channel_id, channel_name, for_signing=True
         )
         timestamp = str(int(time.time()))
-        signature = signing_channel.sign_data(unhexlify(hexdata), timestamp)
+        signature = signing_channel.sign_data(unhexlify(str(hexdata)), timestamp)
         return {
             'signature': signature,
             'signing_ts': timestamp

lbry/wallet/orchstr8/__init__.py

@@ -1,5 +1,5 @@
 __hub_url__ = (
-    "https://github.com/lbryio/hub/releases/download/range-queries/hub"
+    "https://github.com/lbryio/hub/releases/download/v0.2021.12.18.1/hub"
 )
 from .node import Conductor
 from .service import ConductorService

lbry/wallet/server/db/elasticsearch/constants.py

@@ -27,6 +27,19 @@
                     "max_chars": 10
                 }
             },
+            "sd_hash": {
+                "fields": {
+                    "keyword": {
+                        "ignore_above": 96,
+                        "type": "keyword"
+                    }
+                },
+                "type": "text",
+                "index_prefixes": {
+                    "min_chars": 1,
+                    "max_chars": 4
+                }
+            },
             "height": {"type": "integer"},
             "claim_type": {"type": "byte"},
             "censor_type": {"type": "byte"},
@@ -52,14 +65,14 @@
     'timestamp', 'creation_timestamp',
     'duration', 'release_time',
     'tags', 'languages', 'has_source', 'reposted_claim_type',
-    'reposted_claim_id', 'repost_count',
+    'reposted_claim_id', 'repost_count', 'sd_hash',
     'trending_score', 'tx_num'
 }
 
 TEXT_FIELDS = {'author', 'canonical_url', 'channel_id', 'description', 'claim_id', 'censoring_channel_id',
                'media_type', 'normalized_name', 'public_key_bytes', 'public_key_id', 'short_url', 'signature',
                'claim_name', 'signature_digest', 'title', 'tx_id', 'fee_currency', 'reposted_claim_id',
-               'tags'}
+               'tags', 'sd_hash'}
 
 RANGE_FIELDS = {
     'height', 'creation_height', 'activation_height', 'expiration_height',

lbry/wallet/server/db/elasticsearch/search.py

@@ -559,15 +559,15 @@ def expand_query(**kwargs):
                     value = [item[::-1].hex() for item in value]
                 else:
                     value = value[::-1].hex()
-            if not many and key in ('_id', 'claim_id') and len(value) < 20:
+            if not many and key in ('_id', 'claim_id', 'sd_hash') and len(value) < 20:
                 partial_id = True
             if key in ('signature_valid', 'has_source'):
                 continue  # handled later
             if key in TEXT_FIELDS:
                 key += '.keyword'
             ops = {'<=': 'lte', '>=': 'gte', '<': 'lt', '>': 'gt'}
             if partial_id:
-                query['must'].append({"prefix": {"claim_id": value}})
+                query['must'].append({"prefix": {key: value}})
             elif key in RANGE_FIELDS and isinstance(value, str) and value[0] in ops:
                 operator_length = 2 if value[:2] in ops else 1
                 operator, value = value[:operator_length], value[operator_length:]

lbry/wallet/server/leveldb.py

@@ -625,6 +625,7 @@ def _prepare_claim_metadata(self, claim_hash: bytes, claim: ResolveResult):
             'claim_type': CLAIM_TYPES[metadata.claim_type],
             'has_source': reposted_has_source if metadata.is_repost else (
                 False if not metadata.is_stream else metadata.stream.has_source),
+            'sd_hash': metadata.stream.source.sd_hash if metadata.is_stream and metadata.stream.has_source else None,
             'stream_type': STREAM_TYPES[guess_stream_type(metadata.stream.source.media_type)]
                 if metadata.is_stream and metadata.stream.has_source
                 else reposted_stream_type if metadata.is_repost else 0,

tests/integration/claims/test_claim_commands.py

@@ -208,6 +208,10 @@ async def test_basic_claim_search(self):
         await self.assertFindsClaims([three, two], claim_ids=[self.get_claim_id(three), self.get_claim_id(two)])
         await self.assertFindsClaims([three], claim_id=self.get_claim_id(three))
         await self.assertFindsClaims([three], claim_id=self.get_claim_id(three), text='*')
+        # resolve by sd hash
+        two_sd_hash = two['outputs'][0]['value']['source']['sd_hash']
+        await self.assertFindsClaims([two], sd_hash=two_sd_hash)
+        await self.assertFindsClaims([two], sd_hash=two_sd_hash[:2])
 
     async def test_source_filter(self):
         channel = await self.channel_create('@abc')
@@ -1229,6 +1233,8 @@ async def test_sign_hex_encoded_data(self):
         signature2 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign))
         self.assertTrue(verify(channel, unhexlify(data_to_sign), signature1))
         self.assertTrue(verify(channel, unhexlify(data_to_sign), signature2))
+        signature3 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=99))
+        self.assertTrue(verify(channel, unhexlify('99'), signature3))
 
     async def test_channel_export_import_before_sending_channel(self):
         # export