Issue-10: Replace naive implementation with apriori

Author: PiotrZakrzewski

mbdiff/apriori.py

@@ -0,0 +1,37 @@
+from efficient_apriori import apriori
+import numpy as np
+
+def df_to_apriori(df):
+    """Convert pandas df to a data structure understood by efficient apriori."""
+    [tuple(row) for row in df.values.tolist()]
+    res = []
+    for _, row in df.iterrows():
+        transation = []
+        for col in df.columns:
+            val = row[col]
+            if val is np.nan:
+                continue
+            attr_record = f"{col}:{val}"
+            transation.append(attr_record)
+        res.append(tuple(transation))
+    return res
+
+
+def format_res(apriori_results):
+    """Convert apriori results to what Mb Diff expects."""
+    explanations = [rule.lhs for rule in apriori_results]
+    dict_form_explanations = []
+    for explanation in explanations:
+        d = {}
+        for term in explanation:
+            col, value = term.split(":")
+            d[col] = value
+        dict_form_explanations.append(d)
+    return dict_form_explanations
+
+
+def explain(df, min_support, max_order):
+    data = df_to_apriori(df)
+    _, rules = apriori(data, min_support, max_length=max_order)
+    outlier_explanations = [r for r in rules if "outlier:outlier" in r.rhs]
+    return format_res(outlier_explanations)

mbdiff/diff.py

@@ -1,8 +1,10 @@
+from typing import List
 from pandas import DataFrame, read_csv
 from mbdiff.diff_query import DiffQuery, InvalidQuery
 from mbdiff.risk_ratio import risk_ratio
 from mbdiff.attribute_mining import get_combs
 from numpy import nan
+from mbdiff.apriori import explain as aprio_explain
 
 
 def diff_file(
@@ -26,11 +28,32 @@ def diff(
 ):
     """Return explanations, invalid and below support criterium attribute combinations."""
     query.mark_groups(df)
-    ignored_cols = ["outlier"]
+    ignored_cols = []
     # ignore all non categorical columns
     for i, column in enumerate(df.columns):
         if df.dtypes[i] != "object":
             ignored_cols.append(column)
+    sel_cols = [col for col in df.columns if col not in ignored_cols]
+    df = df[sel_cols]
+    return _apriori_diff(df, max_order, min_support, min_risk, ignored_cols)
+
+
+def _apriori_diff(df: DataFrame, max_order: int, min_support: float, min_risk: float, ignored_cols: List):
+    """Explanation mining from attributes close to macrobase DIFF, with use of Apriori."""
+    explanations = aprio_explain(df, min_support, max_order)
+    results, invalid = [], []
+    for explanation in explanations:
+        rr = risk_ratio(explanation, df)
+        res = (rr, explanation)
+        if rr is nan or rr <= min_risk:
+            invalid.append(explanation)
+        else:
+            results.append(res)
+    return results, invalid
+
+
+def _naive_impl(df: DataFrame, max_order: int, min_support: float, min_risk: float, ignored_cols: List):
+    """My own naive implementation of attribute mining, akin to apriori but without minimization."""
     combinations = get_combs(df, max_order, min_support, ignored_cols)
     results, invalid = [], []
     for combination in combinations:

mbdiff/explanation.py

@@ -0,0 +1,15 @@
+"""Module for abstracing away different implementation of explanation finding."""
+from typing import List
+
+
+class Explanation:
+    """Representation of a single explanation."""
+
+    def __init__(self, attrs:List[str], support, score) -> None:
+        self.attrs = attrs
+        self.support = support
+        self.score = score
+    
+    def __str__(self) -> str:
+        attrs = "\t".join(self.attrs)
+        return f"{self.score}\t{self.support}\t" + attrs

requirements.txt

@@ -19,3 +19,4 @@ six==1.15.0
 toml==0.10.2
 typed-ast==1.4.1
 typing-extensions==3.7.4.3
+efficient-apriori==1.1.1