Alpha 3 Merge (#1)

* Changes project name and adds namespace.

* Adds gz reader and bgzf writer.

* Updates readme.

* Adds bgzf support.

* Fixes test commands.

* Adds ogzbuf and updates target names.

* Minor changes.

* Updates bgz seek test.

* Example of actual bgzf.
This commit is a temporary untested example of actual bgzf. I find the the encoding of compressed block size into the header a counterproductive solution to a problem that does not exist. So this code will be replaced with a simpler version of blocked gzip that only restricts the uncompressed data size as opposed to both uncompressed and compressed.

* Reverting actual bgzf implementation.

* Changes naming structure.

* Checks in generic istream.

* Adds generic istream tests.

* Updates readme with new naming structure.

* Fixes bgzf virtual offset bugs.

* Adds initial zstd support.

* Fixes build error.

* Fixes zlib flush issue.

* Adds zstd seek support.

* Fixes move bug in zstd::ibuf.

* Adds zstd.cmake.

* Addresses gcc bug 54316.

* Addresses gcc bug 54316.

* Fixes premature end of file.

* Removes unnecessary assert.

* Adds cmake 3.2 support.

* Attempts to fix cmake issues.

* Attempts to fix cmake issues.

* Attempts to fix cmake issues.

* Attempts to fix cmake issues.

* Attempts to fix cmake issues.

* Adds optional compression level parameter to zstd::obuf.

* Prevents zstd shared libs from being built since advanced api is required.

.travis.yml

@@ -2,6 +2,7 @@ language: cpp
 compiler: gcc
 dist: trusty
 sudo: required
+group: edge
 branches:
   only:
     - develop
@@ -15,12 +16,19 @@ addons:
       - cmake
       - gcc-5
       - g++-5
-#install:
+      - python-dev
+      - python-pip
+install:
+   - sudo pip install cget
+   - cget install -f ./requirements.txt
+#   - curl -L https://github.com/facebook/zstd/archive/v1.2.0.tar.gz > zstd-v1.2.0.tar.gz
+#   - tar -xzf zstd-v1.2.0.tar.gz
+#   - cd zstd-1.2.0/ && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_BUILD_STATIC=1 build/cmake && make && sudo make install && cd ..
 #  - sudo apt-get install --yes --only-upgrade cmake-data
 #  - sudo apt-get install --yes --only-upgrade cmake
 script:
   - cmake --version
   - mkdir build && cd build
-  - cmake -DCMAKE_C_COMPILER=/usr/bin/gcc-5 -DCMAKE_CXX_COMPILER=/usr/bin/g++-5 .. || echo ""
+  - cmake -DCMAKE_TOOLCHAIN_FILE=../cget/cget/cget.cmake -DCMAKE_C_COMPILER=/usr/bin/gcc-5 -DCMAKE_CXX_COMPILER=/usr/bin/g++-5 ..
   - make
   - make CTEST_OUTPUT_ON_FAILURE=1 test

CMakeLists.txt

@@ -1,45 +1,76 @@
 cmake_minimum_required(VERSION 3.2)
 include(ExternalProject)
 include(CMakePackageConfigHelpers)
-project(xzbuf VERSION 1.0.0)
+project(shrinkwrap VERSION 1.0.0)
 
 enable_testing()
 
 set(CMAKE_CXX_STANDARD 11)
 
 if (BUILD_SHARED_LIBS)
     set(LIBLZMA_LIB_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}lzma${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(ZLIB_LIB_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}z${CMAKE_SHARED_LIBRARY_SUFFIX})
 else()
     set(LIBLZMA_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}lzma${CMAKE_STATIC_LIBRARY_SUFFIX})
+    set(ZLIB_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}z${CMAKE_STATIC_LIBRARY_SUFFIX})
 endif()
 
 find_library(LIBLZMA_LIBRARIES
              NAMES ${LIBLZMA_LIB_NAME})
 
+find_library(ZLIB_LIBRARIES
+             NAMES ${ZLIB_LIB_NAME})
+
+#ZSTD can only likn statically since experimental functions are being used.
+find_library(ZSTD_LIBRARIES
+             NAMES ${CMAKE_STATIC_LIBRARY_PREFIX}zstd${CMAKE_STATIC_LIBRARY_SUFFIX})
+
 if (NOT LIBLZMA_LIBRARIES)
     message(FATAL_ERROR "lzma library not found")
 endif()
 
-add_library(xzbuf INTERFACE)
-target_sources(xzbuf INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include/xzbuf.hpp>)
-target_include_directories(xzbuf INTERFACE
-                           $<INSTALL_INTERFACE:include>
-                           $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>)
-target_link_libraries(xzbuf INTERFACE ${LIBLZMA_LIBRARIES})
+if (NOT ZLIB_LIBRARIES)
+    message(FATAL_ERROR "zlib library not found")
+endif()
 
-add_executable(xzbuf-test src/test.cpp)
-target_link_libraries(xzbuf-test xzbuf)
+if (NOT ZSTD_LIBRARIES)
+    message(FATAL_ERROR "zstd library not found")
+endif()
 
-add_test(seek_test xzbuf-test seek)
-add_test(iterator_test xzbuf-test iterator)
+add_library(shrinkwrap INTERFACE)
+if (CMAKE_VERSION VERSION_GREATER 3.3)
+    target_sources(shrinkwrap INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include/shrinkwrap/xz.hpp;${CMAKE_CURRENT_SOURCE_DIR}/include/shrinkwrap/gz.hpp;${CMAKE_CURRENT_SOURCE_DIR}/include/shrinkwrap/zstd.hpp;${CMAKE_CURRENT_SOURCE_DIR}/include/shrinkwrap/istream.hpp>)
+    target_include_directories(shrinkwrap INTERFACE
+                               $<INSTALL_INTERFACE:include>
+                               $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>)
+    target_link_libraries(shrinkwrap INTERFACE ${LIBLZMA_LIBRARIES} ${ZLIB_LIBRARIES} ${ZSTD_LIBRARIES})
+
+    add_executable(shrinkwrap-test src/test.cpp)
+    target_link_libraries(shrinkwrap-test shrinkwrap)
+else()
+    add_executable(shrinkwrap-test src/test.cpp)
+    target_link_libraries(shrinkwrap-test ${LIBLZMA_LIBRARIES} ${ZLIB_LIBRARIES} ${ZSTD_LIBRARIES})
+    target_include_directories(shrinkwrap-test PUBLIC include)
+endif()
 
-install(FILES $<TARGET_PROPERTY:xzbuf,INTERFACE_SOURCES> DESTINATION include)
-install(TARGETS xzbuf EXPORT xzbuf-config
-        LIBRARY DESTINATION lib
-        ARCHIVE DESTINATION lib)
+add_test(xz_seek_test shrinkwrap-test xz-seek)
+add_test(xz_iterator_test shrinkwrap-test xz-iter)
+add_test(gz_iterator_test shrinkwrap-test gz-iter)
+add_test(bgz_seek_test shrinkwrap-test bgz-seek)
+add_test(bgz_iterator_test shrinkwrap-test bgz-iter)
+add_test(zstd_iterator_test shrinkwrap-test zstd-iter)
+add_test(zstd_seek_test shrinkwrap-test zstd-seek)
+add_test(generic_iterator_test shrinkwrap-test generic-iter)
+add_test(generic_seek_test shrinkwrap-test generic-seek)
 
-install(EXPORT ${PROJECT_NAME}-config DESTINATION share/${PROJECT_NAME})
-write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake COMPATIBILITY SameMajorVersion)
-install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake DESTINATION share/${PROJECT_NAME})
+install(DIRECTORY include/shrinkwrap DESTINATION include)
+if (CMAKE_VERSION VERSION_GREATER 3.3)
+    install(TARGETS shrinkwrap EXPORT ${PROJECT_NAME}-config
+            LIBRARY DESTINATION lib
+            ARCHIVE DESTINATION lib)
 
-export(EXPORT ${PROJECT_NAME}-config)
+    install(EXPORT ${PROJECT_NAME}-config DESTINATION share/${PROJECT_NAME})
+    write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake COMPATIBILITY SameMajorVersion)
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake DESTINATION share/${PROJECT_NAME})
+    export(EXPORT ${PROJECT_NAME}-config)
+endif()

README.md

@@ -1,10 +1,10 @@
-# xzbuf
-A streambuf for xz files.
+# Shrink Wrap
+A std::streambuf wrapper for compression formats.
 
-## ixzbuf with std::istream
+## XZ streambuf with std::istream
 ```c++
 std::array<char, 1024> buf;
-ixzbuf sbuf("file.xz");
+shrinkwrap::xz::ibuf sbuf("file.xz");
 std::istream is(&sbuf);
 is.seekg(-1024, std::ios::end);
 while (is)
@@ -13,28 +13,28 @@ while (is)
   std::cout.write(buf.data(), is.gcount());
 }
 ```
-## ixzbuf with std::istreambuf_iterator
+## XZ streambuf with std::istreambuf_iterator
 ```c++
-ixzbuf sbuf("file.xz");
+shrinkwrap::xz::ibuf sbuf("file.xz");
 for (std::istreambuf_iterator<char> it(&sbuf); it != std::istreambuf_iterator<char>{}; ++it)
   std::cout.put(*it);
 ```
 
-## ixzstream 
+## XZ input stream 
 ```c++
 std::array<char, 1024> buf;
-ixzstream is("file.xz");
+shrinkwrap::xz::ostream is("file.xz");
 while (is)
 {
   is.read(buf.data(), buf.size());
   std::cout.write(buf.data(), is.gcount());
 }
 ```
 
-## oxzstream 
+## XZ output stream 
 ```c++
-std::array<char, 1024 * 1024> buf;
-oxzstream os("file.xz");
+std::vector<char> buf(1024 * 1024);
+shrinkwrap::xz::ostream os("file.xz");
 while (std::cin)
 {
   std::cin.read(buf.data(), buf.size());
@@ -43,5 +43,28 @@ while (std::cin)
 }
 ```
 
+## BGZF (Blocked GNU Zip Format)  
+```c++
+std::array<char, 1024> buf;
+shrinkwrap::bgz::istream is("file.xz");
+is.read(buf.data(), buf.size());
+
+// (gzip_block_position << 16) | relative_uncompressed_offset
+auto virtual_offset = is.tellg();
+is.seekg(virtual_offset);
+```
+
+## Generic input stream
+Generic istream detects file format.
+```c++
+std::array<char, 1024> buf;
+shrinkwrap::istream is("file");
+while (is)
+{
+  is.read(buf.data(), buf.size());
+  std::cout.write(buf.data(), is.gcount());
+}
+```
+
 ## Caveats
-* Does not support files with concatenated streams.
+* Does not support files with concatenated xz streams.

dep/xz.cmake

@@ -3,7 +3,7 @@ project(xz VERSION 5.2.3)
 
 execute_process(COMMAND ./configure --prefix=${CMAKE_INSTALL_PREFIX} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
-add_custom_target(xz ALL COMMAND make WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/src/liblzma/.libs/liblzma.a COMMENT "Builing xz ...")
+add_custom_target(xz ALL COMMAND make WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Builing xz ...")
 
 install(DIRECTORY src/liblzma/api/ DESTINATION include FILES_MATCHING PATTERN "*.h")
 

dep/zstd.cmake

@@ -0,0 +1,71 @@
+# ################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
+
+PROJECT(zstd)
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
+OPTION(ZSTD_BUILD_STATIC "must be static" ON)
+SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/build/cmake/CMakeModules")
+
+#-----------------------------------------------------------------------------
+# Add extra compilation flags
+#-----------------------------------------------------------------------------
+INCLUDE(AddZstdCompilationFlags)
+ADD_ZSTD_COMPILATION_FLAGS()
+
+#-----------------------------------------------------------------------------
+# Options
+#-----------------------------------------------------------------------------
+OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
+IF (UNIX)
+    OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
+ELSE (UNIX)
+    OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" OFF)
+ENDIF (UNIX)
+OPTION(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
+OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
+OPTION(ZSTD_BUILD_TESTS "BUILD TESTS" OFF)
+
+IF (ZSTD_LEGACY_SUPPORT)
+    MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!")
+    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=4)
+ELSE (ZSTD_LEGACY_SUPPORT)
+    MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
+    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
+ENDIF (ZSTD_LEGACY_SUPPORT)
+
+#-----------------------------------------------------------------------------
+# Add source directories
+#-----------------------------------------------------------------------------
+SET(ZSTD_BUILD_SHARED OFF CACHE BOOL "must be static" FORCE)
+ADD_SUBDIRECTORY(build/cmake/lib)
+
+IF (ZSTD_BUILD_PROGRAMS)
+    ADD_SUBDIRECTORY(build/cmake/programs)
+ENDIF (ZSTD_BUILD_PROGRAMS)
+
+IF (ZSTD_BUILD_TESTS)
+    IF (NOT ZSTD_BUILD_STATIC)
+        MESSAGE(SEND_ERROR "You need to build static library to build tests")
+    ENDIF (NOT ZSTD_BUILD_STATIC)
+
+    ADD_SUBDIRECTORY(build/cmake/tests)
+ENDIF (ZSTD_BUILD_TESTS)
+
+IF (ZSTD_BUILD_CONTRIB)
+    ADD_SUBDIRECTORY(build/cmake/contrib)
+ENDIF (ZSTD_BUILD_CONTRIB)
+
+#-----------------------------------------------------------------------------
+# Add clean-all target
+#-----------------------------------------------------------------------------
+ADD_CUSTOM_TARGET(clean-all
+   COMMAND ${CMAKE_BUILD_TOOL} clean
+   COMMAND rm -rf ${CMAKE_BINARY_DIR}/
+)