Merge pull request #313 from pysam-developers/AH-htslib-config-query

Ah htslib config query

MANIFEST.in

@@ -24,6 +24,8 @@ include samtools/*/*.h
 include htslib/*.c
 include htslib/*.h
 exclude htslib/config.h
+include htslib/Makefile
+include htslib/htslib_vars.mk
 include htslib/configure
 include htslib/config.mk.in
 include htslib/config.h.in

htslib/Makefile

@@ -224,6 +224,11 @@ libhts.a: $(LIBHTS_OBJS)
 	$(AR) -rc $@ $(LIBHTS_OBJS)
 	-$(RANLIB) $@
 
+print-config:
+	@echo LDFLAGS = $(LDFLAGS)
+	@echo LIBHTS_OBJS = $(LIBHTS_OBJS)
+	@echo LIBS = $(LIBS)
+	@echo PLATFORM = $(PLATFORM)
 
 # The target here is libhts.so, as that is the built file that other rules
 # depend upon and that is used when -lhts appears in other program's recipes.
@@ -420,6 +425,7 @@ force:
 
 .PHONY: all check clean distclean distdir force
 .PHONY: install install-pkgconfig installdirs lib-shared lib-static
-.PHONY: maintainer-clean mostlyclean plugins print-version tags test testclean
+.PHONY: maintainer-clean mostlyclean plugins print-version print-config
+.PHONY: tags test testclean
 .PHONY: clean-so install-so
 .PHONY: clean-dylib install-dylib

pysam/calignmentfile.pyx

@@ -106,16 +106,17 @@ KNOWN_HEADER_FIELDS = {"HD" : {"VN" : str, "SO" : str, "GO" : str},
                        "PG" : {"ID" : str, "PN" : str, "CL" : str, 
                                "PP" : str, "DS" : str, "VN" : str,},}
 
-# output order of fields within records
+# output order of fields within records. Ensure that CL is at
+# the end as parsing a CL will ignore any subsequent records.
 VALID_HEADER_ORDER = {"HD" : ("VN", "SO", "GO"),
                       "SQ" : ("SN", "LN", "AS", "M5",
                                "UR", "SP"),
-                      "RG" : ("ID", "SM", "LB", "DS", 
-                              "PU", "PI", "CN", "DT",
+                      "RG" : ("ID", "CN", "SM", "LB",
+                              "PU", "PI", "DT", "DS",
                               "PL", "FO", "KS", "PG",
                               "PM"),
-                      "PG" : ("PN", "ID", "VN", "CL", 
-                              "PP"),}
+                      "PG" : ("PN", "ID", "VN", "PP",
+                              "DS", "CL"),}
 
 
 def build_header_line(fields, record):

pysam/version.py

@@ -1,6 +1,6 @@
 # pysam versioning information
 
-__version__ = "0.9.1.3"
+__version__ = "0.9.1.4"
 
 __samtools_version__ = "1.3.1"
 

setup.py

@@ -46,6 +46,29 @@ def changedir(path):
         os.chdir(save_dir)
 
 
+def run_configure(option):
+    try:
+        retcode = subprocess.call(
+            " ".join(("./configure", option)),
+            shell=True)
+        if retcode != 0:
+            return False
+        else:
+            return True
+    except OSError as e:
+        return False
+
+
+def run_make_print_config():
+    stdout = subprocess.check_output(["make", "print-config"])
+    if IS_PYTHON3:
+        stdout = stdout.decode("ascii")
+
+    result = dict([[x.strip() for x in line.split("=")]
+                   for line in stdout.splitlines()])
+    return result
+
+
 def configure_library(library_dir, env_options=None, options=[]):
 
     configure_script = os.path.join(library_dir, "configure")
@@ -54,18 +77,6 @@ def configure_library(library_dir, env_options=None, options=[]):
         raise ValueError(
             "configure script {} does not exist".format(configure_script))
 
-    def run_configure(option):
-        try:
-            retcode = subprocess.call(
-                " ".join(("./configure", option)),
-                shell=True)
-            if retcode != 0:
-                return False
-            else:
-                return True
-        except OSError as e:
-            return False
-
     with changedir(library_dir):
         if env_options is not None:
             if run_configure(env_options):
@@ -74,6 +85,7 @@ def run_configure(option):
         for option in options:
             if run_configure(option):
                 return option
+
     return None
 
 
@@ -177,7 +189,8 @@ def distutils_dir_name(dname):
     htslib_configure_options = configure_library(
         "htslib",
         HTSLIB_CONFIGURE_OPTIONS,
-        ["--enable-libcurl"])
+        ["--enable-libcurl",
+         "--disable-libcurl"])
 
     HTSLIB_SOURCE = "builtin"
     print ("# pysam: htslib configure options: {}".format(
@@ -192,6 +205,23 @@ def distutils_dir_name(dname):
             outf.write(
                 "/* conservative compilation options */\n")
 
+    with changedir("htslib"):
+        htslib_make_options = run_make_print_config()
+
+    for key, value in htslib_make_options.items():
+        print ("# pysam: htslib_config {}={}".format(key, value))
+
+    external_htslib_libraries = ['z']
+    if "LIBS" in htslib_make_options:
+        external_htslib_libraries.extend(
+            [re.sub("^-l", "", x) for x in htslib_make_options["LIBS"].split(" ") if x.strip()])
+
+    shared_htslib_sources = [re.sub("\.o", ".c", os.path.join("htslib", x))
+                             for x in
+                             htslib_make_options["LIBHTS_OBJS"].split(" ")]
+
+    htslib_sources = []
+
 if HTSLIB_LIBRARY_DIR:
     # linking against a shared, externally installed htslib version, no
     # sources required for htslib
@@ -206,36 +236,24 @@ def distutils_dir_name(dname):
 elif HTSLIB_MODE == 'separate':
     # add to each pysam component a separately compiled
     # htslib
-    htslib_sources = [
-        x for x in
-        glob.glob(os.path.join("htslib", "*.c")) +
-        glob.glob(os.path.join("htslib", "cram", "*.c"))
-        if x not in EXCLUDE["htslib"]]
+    htslib_sources = shared_htslib_sources
     shared_htslib_sources = htslib_sources
     htslib_library_dirs = []
     htslib_include_dirs = ['htslib']
     internal_htslib_libraries = []
-    external_htslib_libraries = ['z']
 
 elif HTSLIB_MODE == 'shared':
-
     # link each pysam component against the same
     # htslib built from sources included in the pysam
     # package.
-    htslib_sources = []
-    shared_htslib_sources = [
-        x for x in
-        glob.glob(os.path.join("htslib", "*.c")) +
-        glob.glob(os.path.join("htslib", "cram", "*.c"))
-        if x not in EXCLUDE["htslib"]]
     htslib_library_dirs = [
         'pysam',
         ".",
-        os.path.join("build", distutils_dir_name("lib"),
+        os.path.join("build",
+                     distutils_dir_name("lib"),
                      "pysam")]
 
     htslib_include_dirs = ['htslib']
-    external_htslib_libraries = ['z']
 
     if IS_PYTHON3:
         if sys.version_info.minor >= 5:
@@ -277,32 +295,6 @@ def distutils_dir_name(dname):
                 outf.write("{} = {}\n".format(key, config_values[key]))
                 print ("# pysam: config_option: {}={}".format(key, config_values[key]))
 
-if HTSLIB_SOURCE == "builtin":
-    EXCLUDE_HTSLIB = ["htslib/hfile_libcurl.c"]
-    exclude_libcurl = False
-    if htslib_configure_options is None:
-        print ("# pysam: could not configure htslib, choosing "
-               "conservative defaults")
-        exclude_libcurl = True
-    elif "--disable-libcurl" in htslib_configure_options:
-        print ("# pysam: libcurl has been disabled through configure options")
-        exclude_libcurl = True
-    elif config_values["HAVE_HMAC"] == 0 or config_values["HAVE_LIBCURL"] == 0:
-        print ("# pysam: libcurl has not been found - disabled")
-        exclude_libcurl = True
-    else:
-        print ("# pysam: libcurl is enabled")
-
-    if exclude_libcurl:
-        htslib_sources = [x for x in htslib_sources
-                          if x not in EXCLUDE_HTSLIB]
-        shared_htslib_sources = [x for x in shared_htslib_sources
-                                 if x not in EXCLUDE_HTSLIB]
-    elif "--enable-libcurl" in htslib_configure_options:
-        print ("# pysam: libcurl of builtin htslib has been enabled, "
-               "adding shared libcurl and libcrypto")
-        external_htslib_libraries.extend(["curl", "crypto"])
-
 # create empty config.h files if they have not been created automatically
 # or created by the user:
 for fn in config_headers:

tests/AlignmentFile_test.py

@@ -9,6 +9,8 @@
 import os
 import shutil
 import sys
+import re
+import copy
 import collections
 import subprocess
 import logging
@@ -23,7 +25,8 @@
 import pysam
 import pysam.samtools
 from TestUtils import checkBinaryEqual, checkURL, \
-    check_samtools_view_equal, checkFieldEqual, force_str
+    check_samtools_view_equal, checkFieldEqual, force_str, \
+    get_temp_filename
 
 
 DATADIR = "pysam_data"
@@ -1395,6 +1398,72 @@ def testRead(self):
         self.assertTrue(data)
 
 
+class TestHeaderWriteRead(unittest.TestCase):
+    header = {'SQ': [{'LN': 1575, 'SN': 'chr1'},
+                     {'LN': 1584, 'SN': 'chr2'}],
+              'RG': [{'LB': 'SC_1', 'ID': 'L1', 'SM': 'NA12891',
+                      'PU': 'SC_1_10', "CN": "name:with:colon"},
+                     {'LB': 'SC_2', 'ID': 'L2', 'SM': 'NA12891',
+                      'PU': 'SC_2_12', "CN": "name:with:colon"}],
+              'PG': [{'ID': 'P1', 'VN': '1.0', 'CL': 'tool'},
+                     {'ID': 'P2', 'VN': '1.1', 'CL': 'tool with in option -R a\tb',
+                      'PP': 'P1'}],
+              'HD': {'VN': '1.0'},
+              'CO': ['this is a comment', 'this is another comment'],
+    }
+
+    def compare_headers(self, a, b):
+        '''compare two headers a and b.
+
+        Ignore M5 and UR field as they are set application specific.
+        '''
+        for ak, av in a.items():
+            self.assertTrue(ak in b, "key '%s' not in '%s' " % (ak, b))
+            self.assertEqual(
+                len(av), len(b[ak]),
+                "unequal number of entries for key {}: {} vs {}"
+                .format(ak, av, b[ak]))
+
+            for row_a, row_b in zip(av, b[ak]):
+                if isinstance(row_b, dict):
+                    for x in ["M5", "UR"]:
+                        try:
+                            del row_b[x]
+                        except KeyError:
+                            pass
+                self.assertEqual(row_a, row_b)
+
+    def check_read_write(self, flag_write, header):
+
+        fn = get_temp_filename()
+        with pysam.AlignmentFile(
+                fn,
+                flag_write,
+                header=header,
+                reference_filename="pysam_data/ex1.fa") as outf:
+            a = pysam.AlignedSegment()
+            a.query_name = "abc"
+            outf.write(a)
+
+        with pysam.AlignmentFile(fn) as inf:
+            read_header = inf.header
+
+        os.unlink(fn)
+        self.compare_headers(header, read_header)
+
+    def test_SAM(self):
+        self.check_read_write("wh", self.header)
+
+    def test_BAM(self):
+        self.check_read_write("wb", self.header)
+
+    def test_CRAM(self):
+        header = copy.copy(self.header)
+        # for CRAM, \t needs to be quoted:
+        header['PG'][1]['CL'] = re.sub(r"\t", r"\\\\t", header['PG'][1]['CL'])
+        self.check_read_write("wc", header)
+
+
 class TestUnmappedReads(unittest.TestCase):
 
     # TODO