diff --git a/README.md b/README.md
index 066addf..4f3966a 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ Unless required by applicable law or agreed to in writing, software distributed
 
 1. This program downloads the complete list of public INSDC sequence accessions and their last updated dates (if available) for the requested type using the ENA Portal API.
 e.g. for sequence:
-https://www.ebi.ac.uk/ena/portal/api/search?dataPortal=ena&result=sequence&fields=accession,last_updated
+https://www.ebi.ac.uk/ena/browser/api/livelist/sequence?fields=accession,last_updated
 
 2. It then compares the new list against a previous list of the same type and generates 2 lists.
     list of new or updated records since the last time
@@ -37,26 +37,24 @@ You need to provide 3 mandatory arguments i.e datatype, previousSnapshot & outpu
 3. outputLocation : Local folder where the new complete report and the 2 change lists are to be created. Ensure there's
     enough disk space available.
     
-4. email : optionally provide an email address to be notified when the process is complete
-
-5. query : optionally provide a query string to filter the contents of your snapshot. Use the Query page in the 
+4. query : optionally provide a query string to filter the contents of your snapshot. Use the Query page in the 
    https://www.ebi.ac.uk/ena/browser/advanced-search wizard to build up the query. This should be the same query for
    every execution of the tool for a specific snapshot. e.g. --query=dataclass="CON"
 
-6. downloadData : (Optional) If value is true, the tool will also fetch the data for the new/updated records and save
+5. downloadData : (Optional) If value is true, the tool will also fetch the data for the new/updated records and save
    them in a .dat file.
 
-7. format : (Optional) Used only if downloadData=true. Request embl flatfile format (default) or fasta format for
+6. format : (Optional) Used only if downloadData=true. Request embl flatfile format (default) or fasta format for
    downloaded data.
 
-8. annotationOnly : (Optional) Used only if downloadData=true and format=embl. Download only the annotations, excluding
+7. annotationOnly : (Optional) Used only if downloadData=true and format=embl. Download only the annotations, excluding
    sequence lines.
 
-9. includeParentAccession : (Optional) Valid only for coding & noncoding. If true,get parent_accession also from API and
+8. includeParentAccession : (Optional) Valid only for coding & noncoding. If true,get parent_accession also from API and
    include in the output list files.
 
 e.g. 1
-java -jar [path]/snapshot-change-lister-1.1.0.jar --dataType=CODING --previousSnapshot=[path]/coding_20210701.tsv --outputLocation=[path] --email=email@email.com
+java -jar [path]/ena-snapshot-tool-1.3.0.jar --dataType=CODING --previousSnapshot=[path]/coding_20210701.tsv --outputLocation=[path]
 
 If this program were run on 2021-08-03, it would create 3 new files in the outputLocation folder.
 
@@ -67,7 +65,7 @@ coding_20210803_new-or-updated.tsv
 coding_20210803_deleted.tsv
 
 e.g. 2
-java -jar [path]/snapshot-change-lister-1.1.0.jar --dataType=SEQUENCE --previousSnapshot=[path]/sequence_20220220.tsv --outputLocation=[path] --query=dataclass="HTG" --downloadData=true --format=embl --annotationOnly=true"
+java -jar [path]/ena-snapshot-tool-1.3.0.jar --dataType=SEQUENCE --previousSnapshot=[path]/sequence_20220220.tsv --outputLocation=[path] --query=dataclass="HTG" --downloadData=true --format=embl --annotationOnly=true"
 
 If this program were run on 2022-02-23, it would create 4 new files in the outputLocation folder.
 
@@ -82,10 +80,10 @@ sequence_20220223_deleted.tsv
 
 Example for running in LSF:
 
-bsub -n 2 -M 10000 -J coding-snapshot-change-lister -o /path/snapshot-changes/output-20211210.log java -jar  snapshot-change-lister-1.0.0.jar --email=email@email.com --dataType=CODING --previousSnapshot=/path/coding_20211028.tsv --outputLocation=/path/
+bsub -n 2 -M 10000 -J coding-ena-snapshot-tool -o /path/snapshot-changes/output-20211210.log java -jar  ena-snapshot-tool-1.3.0.jar --dataType=CODING --previousSnapshot=/path/coding_20211028.tsv --outputLocation=/path/
 
 
 
 # Support
 
-Direct any questions/issues to https://www.ebi.ac.uk/ena/browser/support with snapshot-change-lister in the subject
+Direct any questions/issues to https://www.ebi.ac.uk/ena/browser/support with ena-snapshot-tool in the subject
diff --git a/build.gradle b/build.gradle
index dcefd2f..94c5918 100644
--- a/build.gradle
+++ b/build.gradle
@@ -20,7 +20,7 @@ plugins {
 }
 
 group = 'uk.ac.ebi.ena.dcap'
-version = '1.2.2'
+version = '1.3.0'
 sourceCompatibility = '1.8'
 
 configurations {
@@ -35,6 +35,9 @@ repositories {
 
 dependencies {
     implementation 'org.apache.commons:commons-lang3:3.4'
+    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.16.1'
+    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.16.1'
+    implementation group: 'com.github.davidmoten', name: 'big-sorter', version: '0.1.25'
     implementation group: 'org.apache.commons', name: 'commons-collections4', version: '4.4'
     implementation group: 'commons-io', name: 'commons-io', version: '2.7'
     implementation group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.13'
diff --git a/settings.gradle b/settings.gradle
index 6337a23..a40c831 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -1 +1 @@
-rootProject.name = 'snapshot-change-lister'
+rootProject.name = 'ena-snapshot-tool'
diff --git a/src/main/java/uk/ac/ebi/ena/dcap/scl/MainRunner.java b/src/main/java/uk/ac/ebi/ena/dcap/scl/MainRunner.java
index dfe9cd0..46eb204 100644
--- a/src/main/java/uk/ac/ebi/ena/dcap/scl/MainRunner.java
+++ b/src/main/java/uk/ac/ebi/ena/dcap/scl/MainRunner.java
@@ -18,26 +18,14 @@
 import lombok.NoArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.CommandLineRunner;
-import org.springframework.mail.javamail.JavaMailSender;
-import org.springframework.mail.javamail.MimeMessagePreparator;
 import org.springframework.stereotype.Component;
-import uk.ac.ebi.ena.dcap.scl.model.DataType;
-import uk.ac.ebi.ena.dcap.scl.model.DiffFiles;
 import uk.ac.ebi.ena.dcap.scl.service.MainService;
 
-import javax.mail.Message;
-import javax.mail.MessagingException;
-import javax.mail.internet.InternetAddress;
-import javax.mail.internet.MimeMessage;
-import java.io.File;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
-import java.util.Date;
 
 @Component
 @Slf4j
@@ -55,9 +43,6 @@ public class MainRunner implements CommandLineRunner {
     @Value("${outputLocation}")
     public String outputLocationPath;
 
-    @Value("${email:#{null}}")
-    public String email;
-
     @Value("${query:#{null}}")
     public String query;
 
@@ -76,60 +61,10 @@ public class MainRunner implements CommandLineRunner {
     @Autowired
     private MainService mainService;
 
-    @Autowired
-    JavaMailSender mailSender;
-
     @SneakyThrows
     @Override
     public void run(String... args) {
-        DataType dataType = DataType.valueOf(dataTypeStr.toUpperCase());
-        File prevSnapshot = new File(previousSnapshotPath);
-        assert prevSnapshot.exists();
-        File outputLocation = new File(outputLocationPath);
-        assert outputLocation.canWrite();
-        if (includeParentAccession && !(dataType == DataType.CODING || dataType == DataType.NONCODING)) {
-            throw new IllegalArgumentException("includeParentAccession can be true only for coding & noncoding");
-        }
-
-        String name = dataType.name().toLowerCase() + "_" + DATE_FORMAT.format(new Date());
-        try {
-            File newSnapshot = mainService.writeLatestSnapshot(dataType, outputLocation, name, query,
-                    includeParentAccession);
-            final DiffFiles diffFiles = mainService.compareSnapshots(prevSnapshot, newSnapshot, outputLocation, name);
-            if (downloadData) {
-                mainService.downloadData(diffFiles.getNewOrChangedList(), format, annotationOnly);
-            }
-            if (StringUtils.isNotBlank(email)) {
-                sendMail(email, dataTypeStr + " change lister completed",
-                        "Compared " + prevSnapshot + " & " + newSnapshot + " in " + outputLocation);
-            }
-        } catch (Exception e) {
-            log.error("error:", e);
-            if (StringUtils.isNotBlank(email)) {
-                sendMail(email, dataTypeStr + " change lister failed", ExceptionUtils.getStackTrace(e));
-            }
-        }
+        mainService.fetchSnapshotAndCompare(dataTypeStr, previousSnapshotPath, outputLocationPath, query, includeParentAccession, format, annotationOnly, downloadData);
     }
 
-    public void sendMail(String email, String subject, String body, String... args) throws MessagingException {
-        MimeMessagePreparator preparator = new MimeMessagePreparator() {
-
-            public void prepare(MimeMessage mimeMessage) throws Exception {
-
-                mimeMessage.setRecipient(Message.RecipientType.TO,
-                        new InternetAddress(email));
-                mimeMessage.setFrom(new InternetAddress("datalib@ebi.ac.uk"));
-                mimeMessage.setText(body
-                        + System.lineSeparator() + StringUtils.join(args, " "));
-                mimeMessage.setSubject(subject);
-            }
-        };
-
-        try {
-            mailSender.send(preparator);
-        } catch (Exception ex) {
-            // simply log it and go on...
-            log.error("Error sending email", ex);
-        }
-    }
 }
diff --git a/src/main/java/uk/ac/ebi/ena/dcap/scl/SnapshotChangeListerApplication.java b/src/main/java/uk/ac/ebi/ena/dcap/scl/SnapshotChangeListerApplication.java
index 97f0f40..fda4e8d 100644
--- a/src/main/java/uk/ac/ebi/ena/dcap/scl/SnapshotChangeListerApplication.java
+++ b/src/main/java/uk/ac/ebi/ena/dcap/scl/SnapshotChangeListerApplication.java
@@ -28,11 +28,5 @@ public static void main(String[] args) {
         SpringApplication.run(SnapshotChangeListerApplication.class, args);
     }
 
-    @Bean
-    public JavaMailSender javaMailService() {
-        JavaMailSenderImpl javaMailSender = new JavaMailSenderImpl();
-        javaMailSender.setHost("smtp.ebi.ac.uk");
-        javaMailSender.setPort(25);
-        return javaMailSender;
-    }
+
 }
diff --git a/src/main/java/uk/ac/ebi/ena/dcap/scl/model/DataType.java b/src/main/java/uk/ac/ebi/ena/dcap/scl/model/DataType.java
index e293e70..6bfcf12 100644
--- a/src/main/java/uk/ac/ebi/ena/dcap/scl/model/DataType.java
+++ b/src/main/java/uk/ac/ebi/ena/dcap/scl/model/DataType.java
@@ -16,7 +16,14 @@
 package uk.ac.ebi.ena.dcap.scl.model;
 
 public enum DataType {
+    ANALYSIS,
+    ASSEMBLY,
+    SAMPLE,
     SEQUENCE,
+    STUDY,
+    READ_RUN,
     CODING,
-    NONCODING;
+    NONCODING,
+
+    TLS_SET, TSA_SET, WGS_SET;
 }
diff --git a/src/main/java/uk/ac/ebi/ena/dcap/scl/model/Line.java b/src/main/java/uk/ac/ebi/ena/dcap/scl/model/Line.java
index 1aca03d..bb5f65c 100644
--- a/src/main/java/uk/ac/ebi/ena/dcap/scl/model/Line.java
+++ b/src/main/java/uk/ac/ebi/ena/dcap/scl/model/Line.java
@@ -19,6 +19,7 @@
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 
 import java.text.DateFormat;
@@ -27,6 +28,7 @@
 @Getter
 @AllArgsConstructor
 @EqualsAndHashCode
+@Slf4j
 public class Line {
 
     public static Line POISON = new Line(null, null);
@@ -36,11 +38,16 @@ public class Line {
 
     @SneakyThrows
     public static Line of(String s, DateFormat df) {
-        final String[] split = StringUtils.split(s);
-        if (split.length == 2) {
-            return new Line(split[0], df.parse(split[1]));
-        } else {
-            return new Line(split[0], null);
+        try {
+            final String[] split = StringUtils.split(s);
+            if (split.length == 2) {
+                return new Line(split[0], df.parse(split[1]));
+            } else {
+                return new Line(split[0], null);
+            }
+        } catch (Exception e) {
+            log.error("Error in line:{}", s, e);
+            throw e;
         }
     }
 }
diff --git a/src/main/java/uk/ac/ebi/ena/dcap/scl/service/CountClient.java b/src/main/java/uk/ac/ebi/ena/dcap/scl/service/CountClient.java
new file mode 100644
index 0000000..754ba36
--- /dev/null
+++ b/src/main/java/uk/ac/ebi/ena/dcap/scl/service/CountClient.java
@@ -0,0 +1,69 @@
+package uk.ac.ebi.ena.dcap.scl.service;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+@Component
+@Slf4j
+public class CountClient {
+    static final String PORTAL_API_COUNT_URL = "https://www.ebi.ac.uk/ena/portal/api/count?result=%s&format=json";
+
+    @SneakyThrows
+    public long getCountFromResults(String result, String query) {
+        // Your JSON array as a string
+
+        // Create ObjectMapper
+        ObjectMapper objectMapper = new ObjectMapper();
+
+        // Parse JSON array
+        JsonNode objNode = objectMapper.readTree(getJson(PORTAL_API_COUNT_URL, result, query));
+
+        // Iterate through objects in the array
+        long codingValue = objNode.get("count").asLong();
+        return codingValue;
+    }
+
+    @SneakyThrows
+    private String getJson(String portalApiResultsUrl, String result, String query) {
+
+        String urlStr = String.format(PORTAL_API_COUNT_URL, result);
+        if (StringUtils.isNotBlank(query)) {
+            urlStr += "&query=" + query;
+        }
+        log.info("getting count from:{}", urlStr);
+        // Create URL object
+        URL url = new URL(urlStr);
+
+        // Open a connection
+        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+
+        // Set the request method to GET
+        connection.setRequestMethod("GET");
+
+        // Get the response code
+        int responseCode = connection.getResponseCode();
+
+        // Read the response from the input stream
+        StringBuilder response;
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
+            response = new StringBuilder();
+            String line;
+            while ((line = reader.readLine()) != null) {
+                response.append(line);
+            }
+
+        }
+
+        return response.toString();
+    }
+
+}
diff --git a/src/main/java/uk/ac/ebi/ena/dcap/scl/service/MainService.java b/src/main/java/uk/ac/ebi/ena/dcap/scl/service/MainService.java
index 769c8dc..42bce5b 100644
--- a/src/main/java/uk/ac/ebi/ena/dcap/scl/service/MainService.java
+++ b/src/main/java/uk/ac/ebi/ena/dcap/scl/service/MainService.java
@@ -33,9 +33,11 @@
 import java.nio.file.Paths;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.util.Date;
 import java.util.List;
 import java.util.concurrent.*;
 
+import static uk.ac.ebi.ena.dcap.scl.MainRunner.DATE_FORMAT;
 import static uk.ac.ebi.ena.dcap.scl.model.Line.POISON;
 
 @Service
@@ -60,7 +62,8 @@ public File writeLatestSnapshot(DataType dataType, File outputLocation, String f
     }
 
     @SneakyThrows
-    public DiffFiles compareSnapshots(File previousSnapshot, File latestSnapshot, File outputLocation, String namePrefix) {
+    public DiffFiles compareSnapshots(File previousSnapshot, File latestSnapshot, File outputLocation,
+                                      String namePrefix) {
         log.info("comparing:{} and {}", previousSnapshot.getAbsolutePath(), latestSnapshot.getAbsolutePath());
         File newOrUpdated = new File(outputLocation.getAbsolutePath() + File.separator + namePrefix + "_new-or" +
                 "-updated.tsv");
@@ -151,8 +154,8 @@ private boolean loadToQueues(File snapshot, BlockingQueue<Line> queue) {
         DateFormat LAST_UPDATED_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd");
         long count = 0;
         try {
-            try (BufferedReader reader = new BufferedReader(new FileReader(snapshot))) {
-                String line = reader.readLine();
+            try (BufferedReader snapshotReader = new BufferedReader(new FileReader(snapshot))) {
+                String line = snapshotReader.readLine();
                 if (StringUtils.isNotBlank(line)) {
                     if (line.startsWith("accession")) {
                         // skip
@@ -160,14 +163,15 @@ private boolean loadToQueues(File snapshot, BlockingQueue<Line> queue) {
                         count++;
                         queue.put(Line.of(line, LAST_UPDATED_DATE_FORMAT));
                     }
-                }
-                while ((line = reader.readLine()) != null) {
-                    count++;
-                    if (count % 100000000 == 0) {
-                        log.info("read {} from {}: {}", count, snapshot.getName(), line);
+                    while ((line = snapshotReader.readLine()) != null) {
+                        count++;
+                        if (count % 100000000 == 0) {
+                            log.info("read {} from {}: {}", count, snapshot.getName(), line);
+                        }
+                        queue.put(Line.of(line, LAST_UPDATED_DATE_FORMAT));
                     }
-                    queue.put(Line.of(line, LAST_UPDATED_DATE_FORMAT));
                 }
+
             }
             log.info("{} added. poisoning:{}", count, snapshot.getName());
 
@@ -195,7 +199,8 @@ public void downloadData(File newOrChangedList, String format, boolean annotatio
         List<List<String>> subSets = ListUtils.partition(lines, 1000);
         try (BufferedOutputStream bw = new BufferedOutputStream(new FileOutputStream(dataFile))) {
             subSets.stream().map(s -> StringUtils.join(s, ",")).forEach(s -> {
-                log.info("downloading:{}-{}", StringUtils.substringBefore(s,","), StringUtils.substringAfterLast(s,","));
+                log.info("downloading:{}-{}", StringUtils.substringBefore(s, ","), StringUtils.substringAfterLast(s,
+                        ","));
                 try {
                     String url = req + s;
                     if ("embl".equalsIgnoreCase(format) && annotationOnly) {
@@ -211,4 +216,34 @@ public void downloadData(File newOrChangedList, String format, boolean annotatio
             });
         }
     }
+
+    @SneakyThrows
+    public void fetchSnapshotAndCompare(String dataTypeStr, String previousSnapshotPath, String outputLocationPath,
+                                        String query, boolean includeParentAccession, String format,
+                                        boolean annotationOnly, boolean downloadData) {
+        DataType dataType = DataType.valueOf(dataTypeStr.toUpperCase());
+        File prevSnapshot = new File(previousSnapshotPath);
+        if (!prevSnapshot.exists()) {
+            log.info("Previous snapshot:{} does not exist.", previousSnapshotPath);
+            prevSnapshot.createNewFile();
+        }
+        File outputLocation = new File(outputLocationPath);
+        assert outputLocation.canWrite();
+        if (includeParentAccession && !(dataType == DataType.CODING || dataType == DataType.NONCODING)) {
+            throw new IllegalArgumentException("includeParentAccession can be true only for coding & noncoding");
+        }
+
+        String name = dataType.name().toLowerCase() + "_" + DATE_FORMAT.format(new Date());
+        try {
+            File newSnapshot = writeLatestSnapshot(dataType, outputLocation, name, query,
+                    includeParentAccession);
+            final DiffFiles diffFiles = compareSnapshots(prevSnapshot, newSnapshot, outputLocation, name);
+            if (downloadData) {
+                downloadData(diffFiles.getNewOrChangedList(), format, annotationOnly);
+            }
+        } catch (Exception e) {
+            log.error("error:", e);
+            System.exit(1);
+        }
+    }
 }
diff --git a/src/main/java/uk/ac/ebi/ena/dcap/scl/service/SnapshotClient.java b/src/main/java/uk/ac/ebi/ena/dcap/scl/service/SnapshotClient.java
index 24ccb5f..05df202 100644
--- a/src/main/java/uk/ac/ebi/ena/dcap/scl/service/SnapshotClient.java
+++ b/src/main/java/uk/ac/ebi/ena/dcap/scl/service/SnapshotClient.java
@@ -15,23 +15,22 @@
  ******************************************************************************/
 package uk.ac.ebi.ena.dcap.scl.service;
 
+import com.github.davidmoten.bigsorter.Sorter;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.http.HttpEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import uk.ac.ebi.ena.dcap.scl.model.DataType;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.*;
 import java.net.URL;
+import java.nio.file.Files;
 
 @Component
 @Slf4j
@@ -39,11 +38,15 @@ public class SnapshotClient {
 
     static final String PORTAL_API_URL = "https://www.ebi.ac.uk/ena/portal/api/search?result=%s&fields=%s";
 
+
     static final String LIVELIST_URL = "https://www.ebi.ac.uk/ena/browser/api/livelist/%s?fields=%s";
     static final String PARENT_ACCESSION = "parent_accession";
     static final String ACCESSION = "accession";
     static final String LAST_UPDATED = "last_updated";
 
+    @Autowired
+    CountClient countClient;
+
     private static String getFields(String resultId, boolean includeParentAccession) {
         String defaultFields;
         if (includeParentAccession && resultId.equalsIgnoreCase("coding") ||
@@ -56,6 +59,16 @@ private static String getFields(String resultId, boolean includeParentAccession)
         return defaultFields;
     }
 
+    public static void bigSortFile(File infile, File outfile) {
+
+        Sorter
+                // set both serializer and natural comparator
+                .linesUtf8()
+                .input(infile)
+                .output(outfile)
+                .sort();
+    }
+
     @SneakyThrows
     public File getLatestSnapshot(DataType dataType, File outputFile, String query, boolean includeParentAccession) {
         String req;
@@ -76,14 +89,47 @@ public File getLatestSnapshot(DataType dataType, File outputFile, String query,
         try (CloseableHttpResponse response1 = client.execute(httpGet)) {
             final HttpEntity entity = response1.getEntity();
             if (entity != null) {
-                try (InputStream in = entity.getContent();
-                     OutputStream out = new FileOutputStream(outputFile)) {
+                File unsortedFile = new File(outputFile.getAbsolutePath() + ".unsorted");
+                String header;
+                long count = 0;
+                try (BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent()));
+                     BufferedWriter out = new BufferedWriter(new FileWriter(unsortedFile))) {
+                    header = in.readLine();
                     log.info("writing response");
-                    IOUtils.copyLarge(in, out);
+                    String line = null;
+                    while ((line = in.readLine()) != null) {
+                        count++;
+                        out.write(line + "\n");
+                    }
+                }
+                log.info("records fetched:{}", count);
+                final long countFromResults = countClient.getCountFromResults(dataType.name().toLowerCase(), query);
+                if (count < countFromResults) {
+                    throw new Exception("Fetched record count " + count + "is lower than index count " + countFromResults);
                 }
+                log.info("sorting to:{}", outputFile.getAbsolutePath());
+                bigSortFile(unsortedFile, outputFile);
+                addHeader(header, outputFile);
+                log.info("deleting unsorted file:{}", unsortedFile.getAbsolutePath());
+                Files.delete(unsortedFile.toPath());
             }
         }
-        log.info("finished new snapshot pull from ENA");
+        log.info("finished new {} snapshot IDs pull from ENA", dataType);
         return outputFile;
     }
+
+    @SneakyThrows
+    private void addHeader(String header, File outputFile) {
+        File tempFile = new File(outputFile.getAbsolutePath() + ".temp");
+        try (BufferedReader in = new BufferedReader(new FileReader(outputFile));
+             BufferedWriter out = new BufferedWriter(new FileWriter(tempFile))) {
+            out.write(header + "\n");
+            String line = null;
+            while ((line = in.readLine()) != null) {
+                out.write(line + "\n");
+            }
+        }
+        Files.delete(outputFile.toPath());
+        Files.move(tempFile.toPath(), outputFile.toPath());
+    }
 }
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
index 88c4aa8..7005a9e 100644
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -13,3 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ##############################################################################/
+spring.main.web-application-type=none
diff --git a/src/test/java/uk/ac/ebi/ena/dcap/scl/MainServiceTest.java b/src/test/java/uk/ac/ebi/ena/dcap/scl/MainServiceTest.java
new file mode 100644
index 0000000..bbe1ea2
--- /dev/null
+++ b/src/test/java/uk/ac/ebi/ena/dcap/scl/MainServiceTest.java
@@ -0,0 +1,19 @@
+package uk.ac.ebi.ena.dcap.scl;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import uk.ac.ebi.ena.dcap.scl.service.MainService;
+
+@SpringBootTest(args = {"--dataType=TLS_SET", "--previousSnapshot=tls.tsv", "--outputLocation=. "})
+public class MainServiceTest {
+
+    @Autowired
+    MainService mainRunner;
+
+    @Test
+    public void testLivelist() {
+        mainRunner.fetchSnapshotAndCompare("TLS_SET", "tls.tsv",
+                ".", null, false, "embl", true, false);
+    }
+}
diff --git a/src/test/java/uk/ac/ebi/ena/dcap/scl/SnapshotClientTest.java b/src/test/java/uk/ac/ebi/ena/dcap/scl/SnapshotClientTest.java
new file mode 100644
index 0000000..969871e
--- /dev/null
+++ b/src/test/java/uk/ac/ebi/ena/dcap/scl/SnapshotClientTest.java
@@ -0,0 +1,15 @@
+package uk.ac.ebi.ena.dcap.scl;
+
+import org.junit.jupiter.api.Test;
+import uk.ac.ebi.ena.dcap.scl.service.SnapshotClient;
+
+import java.io.File;
+
+public class SnapshotClientTest {
+
+    @Test
+    public void testSort() {
+        SnapshotClient.bigSortFile(new File("C:\\projects\\ena-snapshot-tool\\build\\libs\\tls_set_20240201.tsv.unsorted"),
+                new File("C:\\projects\\ena-snapshot-tool\\build\\libs\\tls_set_20240201.tsv"));
+    }
+}