From 7600fa04c52f267ef1ff8122c66d85a8b1fcdc50 Mon Sep 17 00:00:00 2001 From: wkrp Date: Mon, 31 Jul 2023 23:12:20 -0400 Subject: [PATCH 1/3] Update scripts to use packer to unpack cache. Retry wiki scrape on error. --- scripts/cache/cache_constants.py | 6 +-- scripts/update/2_cache.sh | 68 ++++++++++++++++---------------- scripts/wiki/wiki_page_text.py | 17 +++++--- 3 files changed, 48 insertions(+), 43 deletions(-) mode change 100755 => 100644 scripts/update/2_cache.sh diff --git a/scripts/cache/cache_constants.py b/scripts/cache/cache_constants.py index 13294fd0d60..8672de35533 100644 --- a/scripts/cache/cache_constants.py +++ b/scripts/cache/cache_constants.py @@ -36,7 +36,7 @@ NPC_DEFINITIONS = dict() OBJECT_DEFINITIONS = dict() -all_cache_items = sorted(Path(config.DATA_CACHE_PATH / "items").glob("*.json"), +all_cache_items = sorted(Path(config.DATA_CACHE_PATH / "item_defs").glob("*.json"), key=lambda path: int(path.stem)) if len(all_cache_items) == 0: print(">>> ERROR: scripts.cache.cache_constants") @@ -46,7 +46,7 @@ data = json.load(f) ITEM_DEFINITIONS[str(data["id"])] = data -all_cache_npcs = sorted(Path(config.DATA_CACHE_PATH / "npcs").glob("*.json"), +all_cache_npcs = sorted(Path(config.DATA_CACHE_PATH / "npc_defs").glob("*.json"), key=lambda path: int(path.stem)) if len(all_cache_npcs) == 0: print(">>> ERROR: scripts.cache.cache_constants") @@ -56,7 +56,7 @@ data = json.load(f) NPC_DEFINITIONS[str(data["id"])] = data -all_cache_objects = sorted(Path(config.DATA_CACHE_PATH / "objects").glob("*.json"), +all_cache_objects = sorted(Path(config.DATA_CACHE_PATH / "object_defs").glob("*.json"), key=lambda path: int(path.stem)) if len(all_cache_objects) == 0: print(">>> ERROR: scripts.cache.cache_constants") diff --git a/scripts/update/2_cache.sh b/scripts/update/2_cache.sh old mode 100755 new mode 100644 index dfb23931e5e..14a4966de82 --- a/scripts/update/2_cache.sh +++ b/scripts/update/2_cache.sh @@ -38,38 +38,38 @@ jar_file=$(ls | grep .shaded.jar) echo -e " > Extracting osrs-cache..." rm -rf $odb/data/cache/cache-data mkdir -p $odb/data/cache/cache-data -java -jar $jar_file unpack $odb/data/cache/osrs-cache $odb/data/cache/cache-data +java -jar $jar_file dump all $odb/data/cache/osrs-cache $odb/data/cache -echo -e ">>> runelite..." -cd $rl -git pull - -echo -e " > Building RuneLite..." -mvn clean -mvn install -DskipTests - -# Find the cache.jar file with current version and bundled with dependencies -# For example: cache-1.5.27-SNAPSHOT-jar-with-dependencies.jar -cd $rl/cache/target -jar_file=$(ls | grep .jar-with-dependencies.) - -# Remove old cache dumps -echo -e " > Removing the old cache dump in osrsbox-db..." -rm -r $odb/data/cache/items/ -rm -r $odb/data/cache/npcs/ -rm -r $odb/data/cache/objects/ - -# Dump the cache -echo -e " > Dumping cache using RuneLite cache tool..." -echo -e "Jarfile: $jar_file" -java -classpath $jar_file net.runelite.cache.Cache \ --cache $odb/data/cache/cache-data \ --items $odb/data/cache/items - -java -classpath $jar_file net.runelite.cache.Cache \ --cache $odb/data/cache/cache-data \ --npcs $odb/data/cache/npcs - -java -classpath $jar_file net.runelite.cache.Cache \ --cache $odb/data/cache/cache-data \ --objects $odb/data/cache/objects +#echo -e ">>> runelite..." +#cd $rl +#git pull +# +#echo -e " > Building RuneLite..." +#mvn clean +#mvn install -DskipTests +# +## Find the cache.jar file with current version and bundled with dependencies +## For example: cache-1.5.27-SNAPSHOT-jar-with-dependencies.jar +#cd $rl/cache/target +#jar_file=$(ls | grep .jar-with-dependencies.) +# +## Remove old cache dumps +#echo -e " > Removing the old cache dump in osrsbox-db..." +#rm -r $odb/data/cache/items/ +#rm -r $odb/data/cache/npcs/ +#rm -r $odb/data/cache/objects/ +# +## Dump the cache +#echo -e " > Dumping cache using RuneLite cache tool..." +#echo -e "Jarfile: $jar_file" +#java -classpath $jar_file net.runelite.cache.Cache \ +#-cache $odb/data/cache/cache-data \ +#-items $odb/data/cache/items +# +#java -classpath $jar_file net.runelite.cache.Cache \ +#-cache $odb/data/cache/cache-data \ +#-npcs $odb/data/cache/npcs +# +#java -classpath $jar_file net.runelite.cache.Cache \ +#-cache $odb/data/cache/cache-data \ +#-objects $odb/data/cache/objects diff --git a/scripts/wiki/wiki_page_text.py b/scripts/wiki/wiki_page_text.py index 28156f4b97c..6703429e8b1 100644 --- a/scripts/wiki/wiki_page_text.py +++ b/scripts/wiki/wiki_page_text.py @@ -53,14 +53,19 @@ def extract_page_wiki_text(self): "format": "json", "page": self.page_title } + wiki_text = None # Perform HTTP GET request - try: - page_data = requests.get(self.base_url, - headers=config.custom_agent, - params=request).json() - except requests.exceptions.RequestException as e: - raise SystemExit(">>> ERROR: Get request error. Exiting.") from e + for attempt in range(10): + try: + page_data = requests.get(self.base_url, + headers=config.custom_agent, + params=request).json() + break + except requests.exceptions.RequestException as e: + raise SystemExit(">>> ERROR: Get request error. Exiting.") from e + except: + print(">>> ERROR: Probably cloudflare 520") try: # Try to extract the wiki text from the HTTP response From 4ce3120c55ef0b2af6d09ca8f3bda31b37132dec Mon Sep 17 00:00:00 2001 From: Vasco Serrano Date: Mon, 14 Aug 2023 12:35:09 +0100 Subject: [PATCH 2/3] EOL file changes --- .gitignore | 5 +---- data/cache/osrs-cache | 2 +- osrsreboxed/__init__.py | 2 +- pyproject.toml | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 85f9d29ee2a..a3406444048 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,6 @@ # osrsbox-db custom additions .vscode/ -data/cache/cache-data/ -data/cache/items/ -data/cache/npcs/ -data/cache/objects/ +data/cache/ data/monsters/monsters-drops-raw data/monsters/monsters-wiki-page-text-processed.json data/items/items-wiki-page-text-processed.json diff --git a/data/cache/osrs-cache b/data/cache/osrs-cache index 645dbf6d8ee..42a7b747a90 160000 --- a/data/cache/osrs-cache +++ b/data/cache/osrs-cache @@ -1 +1 @@ -Subproject commit 645dbf6d8eeb7bd92eae3e499661ecdc9991b32a +Subproject commit 42a7b747a901cd93acf20f9720a79047c2364d0e diff --git a/osrsreboxed/__init__.py b/osrsreboxed/__init__.py index 2aa24fe69f2..bff004f20ec 100644 --- a/osrsreboxed/__init__.py +++ b/osrsreboxed/__init__.py @@ -1 +1 @@ -__version__ = "2.3.13" +__version__ = "2.3.14" diff --git a/pyproject.toml b/pyproject.toml index 02387372ca5..87064f4b1ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "osrsreboxed" -version = "2.3.13" +version = "2.3.14" description = "A complete and up-to-date database of Old School Runescape (OSRS) items, monsters and prayers accessible using a Python API. Up to date version of osrsbox by PH01L" readme="README.md" homepage = "https://github.com/0xNeffarion/osrsreboxed-db" From 9d34710a07ecdfd117af424289a8b96a973625e8 Mon Sep 17 00:00:00 2001 From: Vasco Serrano Date: Mon, 14 Aug 2023 12:58:51 +0100 Subject: [PATCH 3/3] gitignore osrs cache --- .gitignore | 1 + data/cache/osrs-cache | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a3406444048..d8171591e6c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # osrsbox-db custom additions .vscode/ data/cache/ +data/cache/osrs-cache/ data/monsters/monsters-drops-raw data/monsters/monsters-wiki-page-text-processed.json data/items/items-wiki-page-text-processed.json diff --git a/data/cache/osrs-cache b/data/cache/osrs-cache index 42a7b747a90..45ca270eb3c 160000 --- a/data/cache/osrs-cache +++ b/data/cache/osrs-cache @@ -1 +1 @@ -Subproject commit 42a7b747a901cd93acf20f9720a79047c2364d0e +Subproject commit 45ca270eb3cc5bd2a280a338d3561668312559a6