Added basic filtering + sorting to only sync main products

README.md

@@ -47,6 +47,19 @@ The structure of a profile `.yaml` is as follows:
 
 ```yaml
 entity: product
+
+# optional filtering, only applied on export
+filter:
+  # export main products (parentId = NULL) only
+  - type: "equals"
+    field: "parentId"
+    value: null
+
+# optional sorting, only applied on export
+sort:
+  - field: "name"
+    order: "ASC"
+
 # mappings can either be
 # - by entity_path
 # - by key
@@ -66,6 +79,7 @@ mappings:
     key: "gross_price_eur"
   - file_column: "net price EUR"
     key: "net_price_eur"
+
 # optional serialization script, which is called once per entity
 serialize_script: |
   // See https://rhai.rs/book/ for scripting language documentation

profiles/manufacturer.yaml

@@ -1,4 +1,5 @@
 entity: product_manufacturer
+
 mappings:
   - file_column: "identifier"
     entity_path: "id"

profiles/product_required.yaml

@@ -1,5 +1,16 @@
 # Profile that contains all required fields for a product
 entity: product
+
+filter:
+  # export main products (parentId = NULL) only
+  - type: "equals"
+    field: "parentId"
+    value: null
+
+sort:
+  - field: "name"
+    order: "ASC"
+
 mappings:
   - file_column: "id"
     entity_path: "id"
@@ -15,6 +26,7 @@ mappings:
     entity_path: "stock"
   - file_column: "tax id"
     entity_path: "taxId"
+
 serialize_script: |
   // ToDo: add convenience function to lookup default currencyId
   let price = entity.price.find(|p| p.currencyId == "b7d2554b0ce847cd82f3ac9bd1c0dfca");

src/api.rs

@@ -3,6 +3,7 @@ use anyhow::anyhow;
 use reqwest::{Client, Response, StatusCode};
 use serde::{Deserialize, Serialize};
 use serde_json::json;
+use std::collections::BTreeMap;
 use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
 use thiserror::Error;
@@ -109,7 +110,11 @@ impl SwClient {
         Ok(value)
     }
 
-    pub async fn get_total(&self, entity: &str) -> Result<u64, SwApiError> {
+    pub async fn get_total(
+        &self,
+        entity: &str,
+        filter: &[CriteriaFilter],
+    ) -> Result<u64, SwApiError> {
         // entity needs to be provided as kebab-case instead of snake_case
         let entity = entity.replace('_', "-");
 
@@ -126,6 +131,7 @@ impl SwClient {
                 .bearer_auth(access_token)
                 .json(&json!({
                     "limit": 1,
+                    "filter": filter,
                     "aggregations": [
                         {
                           "name": "count",
@@ -159,8 +165,7 @@ impl SwClient {
     pub async fn list(
         &self,
         entity: &str,
-        page: u64,
-        limit: u64,
+        criteria: &Criteria,
     ) -> Result<SwListResponse, SwApiError> {
         let start_instant = Instant::now();
         // entity needs to be provided as kebab-case instead of snake_case
@@ -176,10 +181,7 @@ impl SwClient {
                     self.credentials.base_url, entity
                 ))
                 .bearer_auth(access_token)
-                .json(&json!({
-                    "page": page,
-                    "limit": limit
-                }))
+                .json(criteria)
                 .send()
                 .await?
         };
@@ -318,3 +320,126 @@ pub struct SwListEntity {
     pub r#type: String,
     pub attributes: serde_json::Map<String, serde_json::Value>,
 }
+
+#[derive(Debug, Serialize)]
+pub struct Criteria {
+    pub limit: u64,
+    pub page: u64,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub filter: Vec<CriteriaFilter>,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub sort: Vec<CriteriaSorting>,
+    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+    pub associations: BTreeMap<String, EmptyObject>,
+}
+
+impl Default for Criteria {
+    fn default() -> Self {
+        Self {
+            limit: Self::MAX_LIMIT,
+            page: 1,
+            sort: vec![],
+            filter: vec![],
+            associations: BTreeMap::new(),
+        }
+    }
+}
+
+impl Criteria {
+    /// Maximum limit accepted by the API server
+    pub const MAX_LIMIT: u64 = 500;
+
+    pub fn add_filter(&mut self, filter: CriteriaFilter) {
+        self.filter.push(filter);
+    }
+
+    pub fn add_sorting(&mut self, sorting: CriteriaSorting) {
+        self.sort.push(sorting);
+    }
+
+    pub fn add_association<S: Into<String>>(&mut self, association: S) {
+        self.associations.insert(association.into(), EmptyObject {});
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub struct CriteriaSorting {
+    pub field: String,
+    pub order: CriteriaSortingOrder,
+}
+
+#[derive(Debug, Clone, Copy, Ord, PartialOrd, Eq, PartialEq, Serialize, Deserialize)]
+pub enum CriteriaSortingOrder {
+    #[serde(rename = "ASC")]
+    Ascending,
+    #[serde(rename = "DESC")]
+    Descending,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "lowercase")]
+pub enum CriteriaFilter {
+    Equals {
+        field: String,
+        value: serde_json::Value,
+    },
+}
+
+#[derive(Debug, Serialize)]
+pub struct EmptyObject {
+    // no fields
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::api::{Criteria, CriteriaFilter, CriteriaSorting, CriteriaSortingOrder};
+
+    #[test]
+    fn criteria_serialize_association() {
+        let mut criteria = Criteria {
+            limit: 10,
+            page: 2,
+            ..Default::default()
+        };
+        criteria.add_association("manufacturer");
+        criteria.add_association("cover");
+
+        let json = serde_json::to_string(&criteria).unwrap();
+        assert_eq!(
+            json,
+            "{\"limit\":10,\"page\":2,\"associations\":{\"cover\":{},\"manufacturer\":{}}}"
+        );
+    }
+
+    #[test]
+    fn criteria_serialize_sorting() {
+        let mut criteria = Criteria {
+            limit: 10,
+            page: 2,
+            ..Default::default()
+        };
+        criteria.add_sorting(CriteriaSorting {
+            field: "manufacturerId".to_string(),
+            order: CriteriaSortingOrder::Descending,
+        });
+
+        let json = serde_json::to_string(&criteria).unwrap();
+        assert_eq!(json, "{\"limit\":10,\"page\":2,\"sort\":[{\"field\":\"manufacturerId\",\"order\":\"DESC\"}]}");
+    }
+
+    #[test]
+    fn criteria_serialize_filter() {
+        let mut criteria = Criteria {
+            limit: 10,
+            page: 2,
+            ..Default::default()
+        };
+        criteria.add_filter(CriteriaFilter::Equals {
+            field: "parentId".to_string(),
+            value: serde_json::Value::Null,
+        });
+
+        let json = serde_json::to_string(&criteria).unwrap();
+        assert_eq!(json, "{\"limit\":10,\"page\":2,\"filter\":[{\"type\":\"Equals\",\"field\":\"parentId\",\"value\":null}]}");
+    }
+}

src/config.rs

@@ -1,3 +1,4 @@
+use crate::api::{CriteriaFilter, CriteriaSorting};
 use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -10,6 +11,10 @@ pub struct Credentials {
 #[derive(Debug, Deserialize)]
 pub struct Schema {
     pub entity: String,
+    #[serde(default = "Vec::new")]
+    pub filter: Vec<CriteriaFilter>,
+    #[serde(default = "Vec::new")]
+    pub sort: Vec<CriteriaSorting>,
     pub mappings: Vec<Mapping>,
     #[serde(default = "String::new")]
     pub serialize_script: String,

src/data/export.rs

@@ -1,3 +1,4 @@
+use crate::api::Criteria;
 use crate::data::transform::serialize_entity;
 use crate::SyncContext;
 use std::cmp;
@@ -6,12 +7,15 @@ use tokio::task::JoinHandle;
 
 /// Might block, so should be used with `task::spawn_blocking`
 pub async fn export(context: Arc<SyncContext>) -> anyhow::Result<()> {
-    let mut total = context.sw_client.get_total(&context.schema.entity).await?;
+    let mut total = context
+        .sw_client
+        .get_total(&context.schema.entity, &context.schema.filter)
+        .await?;
     if let Some(limit) = context.limit {
         total = cmp::min(limit, total);
     }
 
-    let chunk_limit = cmp::min(500, total); // 500 is the maximum allowed per API request
+    let chunk_limit = cmp::min(Criteria::MAX_LIMIT, total);
     let mut page = 1;
     let mut counter = 0;
     println!(
@@ -76,10 +80,20 @@ async fn process_request(
         page, context.schema.entity, chunk_limit
     );
     let mut rows: Vec<Vec<String>> = Vec::with_capacity(chunk_limit as usize);
+    let mut criteria = Criteria {
+        page,
+        limit: chunk_limit,
+        sort: context.schema.sort.clone(),
+        filter: context.schema.filter.clone(),
+        ..Default::default()
+    };
+    for association in &context.associations {
+        criteria.add_association(association);
+    }
 
     let response = context
         .sw_client
-        .list(&context.schema.entity, page, chunk_limit)
+        .list(&context.schema.entity, &criteria)
         .await?;
     for entity in response.data {
         let row = serialize_entity(entity, context)?;

src/main.rs

@@ -1,7 +1,7 @@
 use crate::api::SwClient;
 use crate::config::{Credentials, Mapping, Schema};
 use crate::data::{export, import, prepare_scripting_environment, ScriptingEnvironment};
-use anyhow::{anyhow, Context};
+use anyhow::Context;
 use clap::{ArgAction, Parser, Subcommand};
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -79,6 +79,7 @@ pub struct SyncContext {
     pub limit: Option<u64>,
     pub verbose: bool,
     pub scripting_environment: ScriptingEnvironment,
+    pub associations: Vec<String>,
 }
 
 #[tokio::main]
@@ -156,13 +157,17 @@ async fn create_context(
         .await
         .context("No provided schema file not found")?;
     let schema: Schema = serde_yaml::from_str(&serialized_schema)?;
+    let mut associations = vec![];
     for mapping in &schema.mappings {
         if let Mapping::ByPath(by_path) = mapping {
-            if by_path.entity_path.contains('.') || by_path.entity_path.contains('/') {
-                return Err(anyhow!("entity_path currently only supports fields of the entity and no associations, but found '{}'", by_path.entity_path));
+            if let Some((association, _field)) = by_path.entity_path.rsplit_once('.') {
+                associations.push(association.to_owned());
             }
         }
     }
+    if !associations.is_empty() {
+        println!("Detected associations: {:#?}", associations);
+    }
 
     let serialized_credentials = tokio::fs::read_to_string("./.credentials.toml")
         .await
@@ -185,5 +190,6 @@ async fn create_context(
         file,
         limit,
         verbose,
+        associations,
     })
 }