#21 Fix violation calculation performance

.gitignore

@@ -11,3 +11,4 @@ node_modules/
 
 initial-rank
 initial_guess_impact_results
+*/ethereum/result/

research/ethereum/common/adjacency_list_to_graph.py

@@ -5,6 +5,7 @@
 
 import humanize
 import networkx as nx
+import math
 from networkx import DiGraph
 
 from common.tools import human_readable_interval
@@ -33,7 +34,7 @@ def load_edges(data_file: str) -> Edges:
         if cells[2] == "0":
             continue
         edge = (cells[0], cells[1])
-        weight = float(cells[2])
+        weight = math.sqrt(float(cells[2]))
 
         if edge not in edges:
             edges[edge] = weight

research/ethereum/common/calculate_spring_rank.py

@@ -66,7 +66,7 @@ def bicgstab_callback(x):
         iterations += 1
 
     print_with_time("Solving Bx=b equation using 'bicgstab' iterative method")
-    result = scipy.sparse.linalg.bicgstab(B, b, x0=initial_x, callback=bicgstab_callback, tol=1e-2, atol=0)
+    result = scipy.sparse.linalg.bicgstab(B, b, x0=initial_x, callback=bicgstab_callback, tol=1e-3)
 
     if result[1] != 0:
         print_with_time("Can't solve Bx=b")
@@ -105,7 +105,7 @@ def update_rank(graph: DiGraph, rank: Rank, new_edges: List[Edge]) -> (int, Rank
     return iterations, rank
 
 
-def calculate_violations(A: csr_matrix, rank: [float]) -> (float, float):
+def calculate_violations(A: csr_matrix, rank: [float]) -> (float, float, float):
     """
     Calculate number of violations in a graph given SpringRank scores
     A violaton is an edge going from a lower ranked node to a higher ranked one
@@ -123,11 +123,11 @@ def calculate_violations(A: csr_matrix, rank: [float]) -> (float, float):
     # All elements below main triangle is connection from low-ranked node
     # to high-ranked node. So to get all violation just sum all elements
     # below main triangle.
-    viol = scipy.sparse.tril(A_sort, -1).sum()
+    viol = scipy.sparse.tril(A_sort).sum()
 
     m = A_sort.sum()  # All matrix weights sum
 
-    return (viol, viol / m)
+    return (viol, viol / m, m)
 
 
 def calculate_min_violations(A: csr_matrix) -> (float, float):
@@ -142,17 +142,19 @@ def calculate_min_violations(A: csr_matrix) -> (float, float):
         proportion of all edges against minimum violations
     """
 
-    min_viol = 0
-    for i in range(A.shape[0]):
-        for j in range(i + 1, A.shape[0] - 1):
-            if A[i, j] > 0 and A[j, i] > 0:
-                min_viol = min_viol + min(A[i, j], A[j, i])
+    ii, ji, v = scipy.sparse.find(A)  # I,J,V contain the row, column indices, and values of the nonzero entries.
+
+    min_viol = 0.0
+    for e in range(len(v)):  # for all nodes interactions
+        i, j = ii[e], ji[e]
+        if A[i, j] > 0 and A[j, i] > 0:
+            min_viol = min_viol + min(A[i, j], A[j, i])
 
     m = A.sum()
     return (min_viol, min_viol / m)
 
 
-def calculate_system_violated_energy(A: csr_matrix, rank: [float]) -> (float, float):
+def calculate_system_violated_energy(A: csr_matrix, rank: [float]) -> (float, float, float):
     """
     Calculate number of violations in a graph given SpringRank scores
     A violaton is an edge going from a lower ranked node to a higher ranked one
@@ -166,13 +168,13 @@ def calculate_system_violated_energy(A: csr_matrix, rank: [float]) -> (float, fl
     """
     i, j, v = scipy.sparse.find(A)  # I,J,V contain the row indices, column indices, and values of the nonzero entries.
     normed_rank = (rank - min(rank)) / (max(rank) - min(rank))  # normalize
-    wv = 0
+    wv = 0.0
     for e in range(len(v)):  # for all nodes interactions
         if normed_rank[i[e]] < normed_rank[j[e]]:  # compare ranks of two nodes i and j
             wv = wv + v[e] * (normed_rank[j[e]] - normed_rank[i[e]])
 
     H = calculate_Hamiltonion(A, rank)
-    return (wv, wv / H)
+    return (wv, wv / H, H)
 
 
 def calculate_Hamiltonion(A: csr_matrix, rank: [float]) -> float:
@@ -186,9 +188,11 @@ def calculate_Hamiltonion(A: csr_matrix, rank: [float]) -> float:
     """
 
     H = 0.0
+    normed_rank = (rank - min(rank)) / (max(rank) - min(rank))  # normalize
 
-    for i in range(A.shape[0]):
-        for j in range(A.shape[1]):
-            H = H + 0.5 * A[i, j] * (rank[i] - rank[j] - 1) ** 2
+    ii, ji, v = scipy.sparse.find(A)  # I,J,V contain the row, column indices, and values of the nonzero entries.
+    for e in range(len(v)):  # for all nodes interactions
+        i, j = ii[e], ji[e]
+        H = H + 0.5 * A[i, j] * (normed_rank[i] - normed_rank[j] - 1) ** 2
 
     return H

research/ethereum/common/test_calculate_rank.py

@@ -1,7 +1,10 @@
 import unittest
 
+import humanize
+
 from common.calculate_significance import estimate_beta, test_ranks_significance
-from common.calculate_spring_rank import calculate_spring_rank, calculate_Hamiltonion
+from common.calculate_spring_rank import calculate_spring_rank, calculate_Hamiltonion, calculate_violations, \
+    calculate_min_violations, calculate_system_violated_energy
 from common.generate_graph import generate_graph
 
 
@@ -10,7 +13,7 @@ class GenerateGraphTest(unittest.TestCase):
     def test_generate_graph(self):
         print("Generating network")
         origin_beta = 2.1
-        A, origin_raw_rank = generate_graph(N=70, beta=origin_beta, c=1)
+        A, origin_raw_rank = generate_graph(N=40, beta=origin_beta, c=1)
 
         print("Calculating rank")
         iterations, calculated_raw_rank = calculate_spring_rank(A)
@@ -26,6 +29,15 @@ def test_generate_graph(self):
         calculated_energy = calculate_Hamiltonion(A, calculated_raw_rank)
         print(f"Calculated energies: '{calculated_from_origin_energy}' and '{calculated_energy}'")
 
+        print("Calculate violations")
+        v, vp, ws = calculate_violations(A, calculated_raw_rank)
+        mv, mvp = calculate_min_violations(A)
+        ve, vep, H = calculate_system_violated_energy(A, calculated_raw_rank)
+
+        print(f"Violations: {v}[{vp}%] :: min violations: {mv}[{mvp}%]. Sum Aij: {ws}")
+        print(f"Violation energy: {ve}[{vep}%] :: total energy: {H}")
+        print("-----------------------------------------------")
+
         self.assertEqual(True, True)
 
     def test_graph_significance(self):

research/ethereum/initial-calculation-experiment/calculate_ethereum_rank.py

@@ -1,18 +1,17 @@
 import os
 import sys
 
-
 sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
 
 import glob
 from collections import OrderedDict
 
 from common.adjacency_list_to_graph import load_edges, build_graph
-from common.calculate_spring_rank import calculate_spring_rank
+from common.calculate_spring_rank import calculate_spring_rank, calculate_violations, calculate_min_violations, \
+    calculate_system_violated_energy
 from common.graph_to_matrix import build_matrix
 from common.tools import print_with_time
 
-
 """
 Calculate ethereum rank from `../data` folders data.
 """
@@ -44,7 +43,17 @@
 
 print("Storing results")
 initial_rank_file = open("../result/calculated-rank", "w")
-for node, node_rank in rank.items():
+
+for node, node_rank in sorted(rank.items(), key=lambda kv: kv[1], reverse=True):
     initial_rank_file.write(f"{node} {node_rank}\r\n")
 initial_rank_file.close()
 print("-----------------------------------------------")
+
+print("Calculate violations")
+v, vp, ws = calculate_violations(A, raw_rank)
+mv, mvp = calculate_min_violations(A)
+ve, vep, H = calculate_system_violated_energy(A, raw_rank)
+
+print(f"Violations: {v} [{vp}%] :: min violations: {mv} [{mvp}%]. Sum Aij: {ws}")
+print(f"Violation energy: {ve} [{vep}%] :: total energy: {H}")
+print("-----------------------------------------------")