Merge pull request #3 from 0xNeffarion/wkpatrick/master

Wkpatrick/master

.gitignore

@@ -1,9 +1,7 @@
 # osrsbox-db custom additions
 .vscode/
-data/cache/cache-data/
-data/cache/items/
-data/cache/npcs/
-data/cache/objects/
+data/cache/
+data/cache/osrs-cache/
 data/monsters/monsters-drops-raw
 data/monsters/monsters-wiki-page-text-processed.json
 data/items/items-wiki-page-text-processed.json

scripts/cache/cache_constants.py

@@ -36,7 +36,7 @@
 NPC_DEFINITIONS = dict()
 OBJECT_DEFINITIONS = dict()
 
-all_cache_items = sorted(Path(config.DATA_CACHE_PATH / "items").glob("*.json"),
+all_cache_items = sorted(Path(config.DATA_CACHE_PATH / "item_defs").glob("*.json"),
                          key=lambda path: int(path.stem))
 if len(all_cache_items) == 0:
     print(">>> ERROR: scripts.cache.cache_constants")
@@ -46,7 +46,7 @@
         data = json.load(f)
         ITEM_DEFINITIONS[str(data["id"])] = data
 
-all_cache_npcs = sorted(Path(config.DATA_CACHE_PATH / "npcs").glob("*.json"),
+all_cache_npcs = sorted(Path(config.DATA_CACHE_PATH / "npc_defs").glob("*.json"),
                         key=lambda path: int(path.stem))
 if len(all_cache_npcs) == 0:
     print(">>> ERROR: scripts.cache.cache_constants")
@@ -56,7 +56,7 @@
         data = json.load(f)
         NPC_DEFINITIONS[str(data["id"])] = data
 
-all_cache_objects = sorted(Path(config.DATA_CACHE_PATH / "objects").glob("*.json"),
+all_cache_objects = sorted(Path(config.DATA_CACHE_PATH / "object_defs").glob("*.json"),
                            key=lambda path: int(path.stem))
 if len(all_cache_objects) == 0:
     print(">>> ERROR: scripts.cache.cache_constants")

scripts/update/2_cache.sh

@@ -38,38 +38,38 @@ jar_file=$(ls | grep .shaded.jar)
 echo -e "  > Extracting osrs-cache..."
 rm -rf $odb/data/cache/cache-data
 mkdir -p $odb/data/cache/cache-data
-java -jar $jar_file unpack $odb/data/cache/osrs-cache $odb/data/cache/cache-data
+java -jar $jar_file dump all $odb/data/cache/osrs-cache $odb/data/cache
 
-echo -e ">>> runelite..."
-cd $rl
-git pull
-
-echo -e "  > Building RuneLite..."
-mvn clean
-mvn install -DskipTests
-
-# Find the cache.jar file with current version and bundled with dependencies
-# For example: cache-1.5.27-SNAPSHOT-jar-with-dependencies.jar
-cd $rl/cache/target
-jar_file=$(ls | grep .jar-with-dependencies.)
-
-# Remove old cache dumps
-echo -e "  > Removing the old cache dump in osrsbox-db..."
-rm -r $odb/data/cache/items/
-rm -r $odb/data/cache/npcs/
-rm -r $odb/data/cache/objects/
-
-# Dump the cache
-echo -e "  > Dumping cache using RuneLite cache tool..."
-echo -e "Jarfile: $jar_file"
-java -classpath $jar_file net.runelite.cache.Cache \
--cache $odb/data/cache/cache-data \
--items $odb/data/cache/items
-
-java -classpath $jar_file net.runelite.cache.Cache \
--cache $odb/data/cache/cache-data \
--npcs $odb/data/cache/npcs
-
-java -classpath $jar_file net.runelite.cache.Cache \
--cache $odb/data/cache/cache-data \
--objects $odb/data/cache/objects
+#echo -e ">>> runelite..."
+#cd $rl
+#git pull
+#
+#echo -e "  > Building RuneLite..."
+#mvn clean
+#mvn install -DskipTests
+#
+## Find the cache.jar file with current version and bundled with dependencies
+## For example: cache-1.5.27-SNAPSHOT-jar-with-dependencies.jar
+#cd $rl/cache/target
+#jar_file=$(ls | grep .jar-with-dependencies.)
+#
+## Remove old cache dumps
+#echo -e "  > Removing the old cache dump in osrsbox-db..."
+#rm -r $odb/data/cache/items/
+#rm -r $odb/data/cache/npcs/
+#rm -r $odb/data/cache/objects/
+#
+## Dump the cache
+#echo -e "  > Dumping cache using RuneLite cache tool..."
+#echo -e "Jarfile: $jar_file"
+#java -classpath $jar_file net.runelite.cache.Cache \
+#-cache $odb/data/cache/cache-data \
+#-items $odb/data/cache/items
+#
+#java -classpath $jar_file net.runelite.cache.Cache \
+#-cache $odb/data/cache/cache-data \
+#-npcs $odb/data/cache/npcs
+#
+#java -classpath $jar_file net.runelite.cache.Cache \
+#-cache $odb/data/cache/cache-data \
+#-objects $odb/data/cache/objects

scripts/wiki/wiki_page_text.py

@@ -53,14 +53,19 @@ def extract_page_wiki_text(self):
             "format": "json",
             "page": self.page_title
         }
+        wiki_text = None
 
         # Perform HTTP GET request
-        try:
-            page_data = requests.get(self.base_url,
-                                     headers=config.custom_agent,
-                                     params=request).json()
-        except requests.exceptions.RequestException as e:
-            raise SystemExit(">>> ERROR: Get request error. Exiting.") from e
+        for attempt in range(10):
+            try:
+                page_data = requests.get(self.base_url,
+                                         headers=config.custom_agent,
+                                         params=request).json()
+                break
+            except requests.exceptions.RequestException as e:
+                raise SystemExit(">>> ERROR: Get request error. Exiting.") from e
+            except:
+                print(">>> ERROR: Probably cloudflare 520")
 
         try:
             # Try to extract the wiki text from the HTTP response