diff --git a/.gitignore b/.gitignore index 85f9d29ee2a..d8171591e6c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,7 @@ # osrsbox-db custom additions .vscode/ -data/cache/cache-data/ -data/cache/items/ -data/cache/npcs/ -data/cache/objects/ +data/cache/ +data/cache/osrs-cache/ data/monsters/monsters-drops-raw data/monsters/monsters-wiki-page-text-processed.json data/items/items-wiki-page-text-processed.json diff --git a/scripts/cache/cache_constants.py b/scripts/cache/cache_constants.py index 13294fd0d60..8672de35533 100644 --- a/scripts/cache/cache_constants.py +++ b/scripts/cache/cache_constants.py @@ -36,7 +36,7 @@ NPC_DEFINITIONS = dict() OBJECT_DEFINITIONS = dict() -all_cache_items = sorted(Path(config.DATA_CACHE_PATH / "items").glob("*.json"), +all_cache_items = sorted(Path(config.DATA_CACHE_PATH / "item_defs").glob("*.json"), key=lambda path: int(path.stem)) if len(all_cache_items) == 0: print(">>> ERROR: scripts.cache.cache_constants") @@ -46,7 +46,7 @@ data = json.load(f) ITEM_DEFINITIONS[str(data["id"])] = data -all_cache_npcs = sorted(Path(config.DATA_CACHE_PATH / "npcs").glob("*.json"), +all_cache_npcs = sorted(Path(config.DATA_CACHE_PATH / "npc_defs").glob("*.json"), key=lambda path: int(path.stem)) if len(all_cache_npcs) == 0: print(">>> ERROR: scripts.cache.cache_constants") @@ -56,7 +56,7 @@ data = json.load(f) NPC_DEFINITIONS[str(data["id"])] = data -all_cache_objects = sorted(Path(config.DATA_CACHE_PATH / "objects").glob("*.json"), +all_cache_objects = sorted(Path(config.DATA_CACHE_PATH / "object_defs").glob("*.json"), key=lambda path: int(path.stem)) if len(all_cache_objects) == 0: print(">>> ERROR: scripts.cache.cache_constants") diff --git a/scripts/update/2_cache.sh b/scripts/update/2_cache.sh old mode 100755 new mode 100644 index dfb23931e5e..14a4966de82 --- a/scripts/update/2_cache.sh +++ b/scripts/update/2_cache.sh @@ -38,38 +38,38 @@ jar_file=$(ls | grep .shaded.jar) echo -e " > Extracting osrs-cache..." rm -rf $odb/data/cache/cache-data mkdir -p $odb/data/cache/cache-data -java -jar $jar_file unpack $odb/data/cache/osrs-cache $odb/data/cache/cache-data +java -jar $jar_file dump all $odb/data/cache/osrs-cache $odb/data/cache -echo -e ">>> runelite..." -cd $rl -git pull - -echo -e " > Building RuneLite..." -mvn clean -mvn install -DskipTests - -# Find the cache.jar file with current version and bundled with dependencies -# For example: cache-1.5.27-SNAPSHOT-jar-with-dependencies.jar -cd $rl/cache/target -jar_file=$(ls | grep .jar-with-dependencies.) - -# Remove old cache dumps -echo -e " > Removing the old cache dump in osrsbox-db..." -rm -r $odb/data/cache/items/ -rm -r $odb/data/cache/npcs/ -rm -r $odb/data/cache/objects/ - -# Dump the cache -echo -e " > Dumping cache using RuneLite cache tool..." -echo -e "Jarfile: $jar_file" -java -classpath $jar_file net.runelite.cache.Cache \ --cache $odb/data/cache/cache-data \ --items $odb/data/cache/items - -java -classpath $jar_file net.runelite.cache.Cache \ --cache $odb/data/cache/cache-data \ --npcs $odb/data/cache/npcs - -java -classpath $jar_file net.runelite.cache.Cache \ --cache $odb/data/cache/cache-data \ --objects $odb/data/cache/objects +#echo -e ">>> runelite..." +#cd $rl +#git pull +# +#echo -e " > Building RuneLite..." +#mvn clean +#mvn install -DskipTests +# +## Find the cache.jar file with current version and bundled with dependencies +## For example: cache-1.5.27-SNAPSHOT-jar-with-dependencies.jar +#cd $rl/cache/target +#jar_file=$(ls | grep .jar-with-dependencies.) +# +## Remove old cache dumps +#echo -e " > Removing the old cache dump in osrsbox-db..." +#rm -r $odb/data/cache/items/ +#rm -r $odb/data/cache/npcs/ +#rm -r $odb/data/cache/objects/ +# +## Dump the cache +#echo -e " > Dumping cache using RuneLite cache tool..." +#echo -e "Jarfile: $jar_file" +#java -classpath $jar_file net.runelite.cache.Cache \ +#-cache $odb/data/cache/cache-data \ +#-items $odb/data/cache/items +# +#java -classpath $jar_file net.runelite.cache.Cache \ +#-cache $odb/data/cache/cache-data \ +#-npcs $odb/data/cache/npcs +# +#java -classpath $jar_file net.runelite.cache.Cache \ +#-cache $odb/data/cache/cache-data \ +#-objects $odb/data/cache/objects diff --git a/scripts/wiki/wiki_page_text.py b/scripts/wiki/wiki_page_text.py index 28156f4b97c..6703429e8b1 100644 --- a/scripts/wiki/wiki_page_text.py +++ b/scripts/wiki/wiki_page_text.py @@ -53,14 +53,19 @@ def extract_page_wiki_text(self): "format": "json", "page": self.page_title } + wiki_text = None # Perform HTTP GET request - try: - page_data = requests.get(self.base_url, - headers=config.custom_agent, - params=request).json() - except requests.exceptions.RequestException as e: - raise SystemExit(">>> ERROR: Get request error. Exiting.") from e + for attempt in range(10): + try: + page_data = requests.get(self.base_url, + headers=config.custom_agent, + params=request).json() + break + except requests.exceptions.RequestException as e: + raise SystemExit(">>> ERROR: Get request error. Exiting.") from e + except: + print(">>> ERROR: Probably cloudflare 520") try: # Try to extract the wiki text from the HTTP response