boutiques · glatard · Mar 17, 2020 · Mar 11, 2020 · Mar 11, 2020 · Mar 11, 2020
diff --git a/tools/python/boutiques/bosh.py b/tools/python/boutiques/bosh.py
@@ -821,6 +821,44 @@ def data(*params):
 data.__doc__ += parser_dataDelete().format_help()
 
 
+def parser_deprecate():
+    parser = ArgumentParser("Deprecates a published descriptor by creating a"
+                            " new version with the 'deprecated' tag on Zenodo."
+                            " The descriptor remains available from its Zenodo"
+                            "id, but it won't show in search results. This "
+                            "works by creating a new version of the tool in "
+                            "Zenodo, marked with keyword 'deprecated'.")
+    parser.add_argument("zid", action="store", help="Zenodo id "
+                        "of the descriptor to deprecate, "
+                        "prefixed by 'zenodo.', e.g. zenodo.123456")
+    parser.add_argument("--by", action="store", help="Zenodo id (e.g., "
+                        "zenodo-1234) of a  descriptor that will supersede "
+                        "the deprecated one.")
+    parser.add_argument("--zenodo-token", action="store",
+                        help="Zenodo API token to use for authentication. "
+                        "If not used, token will be read from configuration "
+                        "file or requested interactively.")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="Print information messages")
+    parser.add_argument("--sandbox", action="store_true",
+                        help="use Zenodo's sandbox instead of "
+                        "production server. Recommended for tests.")
+    return parser
+
+
+def deprecate(*params):
+    parser = parser_deprecate()
+    result = parser.parse_args(params)
+
+    from boutiques.deprecate import deprecate
+    return deprecate(result.zid, by_zenodo_id=result.by,
+                     sandbox=result.sandbox, verbose=result.verbose,
+                     zenodo_token=result.zenodo_token)
+
+
+deprecate.__doc__ = parser_deprecate().format_help()
+
+
 def parser_bosh():
     parser = ArgumentParser(add_help=False,
                             formatter_class=RawTextHelpFormatter)
@@ -841,6 +879,8 @@ def parser_bosh():
 * test: run pytest on a descriptor detailing tests.
 
 TOOL SEARCH & PUBLICATION
+* deprecate: deprecate a published tool. The tool will still be published and
+usable, but it won't show in search results.
 * publish: create an entry in Zenodo for the descriptor and adds the DOI \
 created by Zenodo to the descriptor.
 * pull: download a descriptor from Zenodo.
@@ -862,7 +902,7 @@ def parser_bosh():
                                     "import", "export", "publish",
                                     "invocation", "evaluate", "test",
                                     "example", "search", "pull",
-                                    "data", "pprint", "version"]))
+                                    "data", "pprint", "version", "deprecate"]))
 
     parser.add_argument("--help", "-h", action="store_true",
                         help="show this help message and exit")
@@ -946,6 +986,9 @@ def bosh_return(val, code=0, hide=False, formatted=None):
         elif func == "version":
             from boutiques.__version__ import VERSION
             return bosh_return(VERSION)
+        elif func == "deprecate":
+            out = deprecate(*params)
+            return bosh_return(out)
         else:
             parser.print_help()
             raise SystemExit

diff --git a/tools/python/boutiques/deprecate.py b/tools/python/boutiques/deprecate.py
@@ -0,0 +1,86 @@
+import json
+import tempfile
+from boutiques.util.utils import loadJson
+from boutiques.puller import Puller
+from boutiques.publisher import Publisher
+from boutiques.logger import print_info, raise_error, print_warning
+from boutiques.zenodoHelper import ZenodoHelper, ZenodoError
+from urllib.request import urlopen, urlretrieve
+
+
+class DeprecateError(Exception):
+    pass
+
+
+def deprecate(zenodo_id, by_zenodo_id=None, sandbox=False, verbose=False,
+              zenodo_token=None, download_function=urlretrieve):
+
+    # Get the descriptor and Zenodo id
+    puller = Puller([zenodo_id], verbose=verbose, sandbox=sandbox)
+    descriptor_fname = puller.pull()[0]
+    descriptor_json = loadJson(descriptor_fname, sandbox=sandbox,
+                               verbose=verbose)
+
+    # Return if tool is already deprecated
+    deprecated = descriptor_json.get('deprecated-by-doi')
+    if deprecated is not None:
+        if isinstance(deprecated, str):
+            print_info('Tool {0} is already deprecated by {1} '
+                       .format(zenodo_id, deprecated))
+        if by_zenodo_id is not None:
+            prompt = ("Tool {0} will be deprecated by {1}, "
+                      "this cannot be undone. Are you sure? (Y/n) ")\
+                          .format(zenodo_id, by_zenodo_id)
+            ret = input(prompt)
+            if ret.upper() != "Y":
+                return
+        else:
+            print_warning('Tool {0} is already deprecated'.format(zenodo_id))
+            return
+    # Set record id and Zenodo id
+    zhelper = ZenodoHelper(sandbox=sandbox, no_int=True, verbose=verbose)
+    zid = zhelper.get_zid_from_filename(descriptor_fname)
+    record_id = zhelper.get_record_id_from_zid(zid)
+
+    # Return if tool has a newer version
+    record = zhelper.zenodo_get_record(record_id)
+    if not record['metadata']['relations']['version'][0]['is_last']:
+        new_version = (record['metadata']['relations']
+                       ['version'][0]['last_child']['pid_value'])
+        raise_error(DeprecateError, 'Tool {0} has a newer version '
+                                    '(zenodo.{1}), it cannot be deprecated.'
+                                    .format(zenodo_id, new_version))
+        return
+
+    # Add deprecated property
+    if by_zenodo_id is None:
+        descriptor_json['deprecated-by-doi'] = True
+    else:
+        # Check that by_zenodo_id exists
+        by_record_id = zhelper.get_record_id_from_zid(by_zenodo_id)
+        if zhelper.record_exists(by_record_id) is False:
+            raise_error(DeprecateError,
+                        "Tool does not exist: {0}".format(by_zenodo_id))
+        # Assign deprecated-by property
+        by_doi_id = zhelper.get_doi_from_zid(by_zenodo_id)
+        descriptor_json['deprecated-by-doi'] = by_doi_id
+
+    # Add doi to descriptor (mandatory for update)
+    if descriptor_json.get('doi') is None:
+        descriptor_json['doi'] = zhelper.get_doi_from_zid(zid)
+
+    # Save descriptor in temp file
+    tmp = tempfile.NamedTemporaryFile(delete=False, mode='w', suffix=".json")
+    content = json.dumps(descriptor_json, indent=4,
+                         sort_keys=True)
+    tmp.write(content)
+    tmp.close()
+
+    # Publish updated descriptor
+    publisher = Publisher(tmp.name, zenodo_token,
+                          replace=True,
+                          sandbox=sandbox,
+                          no_int=True,
+                          id="zenodo."+zid,
+                          verbose=verbose)
+    return publisher.publish()
diff --git a/tools/python/boutiques/publisher.py b/tools/python/boutiques/publisher.py
@@ -104,10 +104,7 @@ def publish(self):
         if(not self.no_int):
             prompt = ("The descriptor will be published to Zenodo, "
                       "this cannot be undone. Are you sure? (Y/n) ")
-            try:
-                ret = raw_input(prompt)  # Python 2
-            except NameError:
-                ret = input(prompt)  # Python 3
+            ret = input(prompt)
             if ret.upper() != "Y":
                 return
 
@@ -134,10 +131,7 @@ def publish(self):
                               "would you like to update it? "
                               "(Y:Update existing / n:Publish new entry with "
                               "name {}) ".format(self.descriptor.get("name")))
-                    try:
-                        ret = raw_input(prompt)  # Python 2
-                    except NameError:
-                        ret = input(prompt)  # Python 3
+                    ret = input(prompt)
                     if ret.upper() == "Y":
                         publish_update = True
                 else:
@@ -157,6 +151,9 @@ def publish(self):
         self.descriptor['doi'] = self.doi
         with open(self.descriptor_file_name, "w") as f:
             f.write(json.dumps(self.descriptor, indent=4))
+        if os.path.isfile(self.descriptor_file_name):
+            return "OK"
+        return False
 
     def create_metadata(self):
         data = {
@@ -180,7 +177,7 @@ def create_metadata(self):
                 if isinstance(value, bool):
                     keywords.append(key)
                 # Tag is of form 'tag-name':'tag-value', it is a key-value pair
-                if self.is_str(value):
+                if isinstance(value, str):
                     keywords.append(key + ":" + value)
                 # Tag is of form 'tag-name': ['value1', 'value2'], it is a
                 # list of key-value pairs
@@ -190,30 +187,30 @@ def create_metadata(self):
             keywords.append(self.descriptor['container-image']['type'])
         if self.descriptor.get('tests'):
             keywords.append('tested')
+        if self.descriptor.get('deprecated-by-doi'):
+            keywords.append('deprecated')
+            if isinstance(self.descriptor['deprecated-by-doi'], str):
+                keywords.append('deprecated-by-doi:' +
+                                self.descriptor['deprecated-by-doi'])
+            self.addRelatedIdentifiers(
+                data, self.descriptor['deprecated-by-doi'],
+                'isPreviousVersionOf')
+
         if self.url is not None:
-            self.addHasPart(data, self.url)
+            self.addRelatedIdentifiers(data, self.url, 'hasPart')
         if self.online_platforms is not None:
             for p in self.online_platforms:
-                self.addHasPart(data, p)
+                self.addRelatedIdentifiers(data, p, 'hasPart')
         if self.tool_doi is not None:
-            self.addHasPart(data, self.tool_doi)
+            self.addRelatedIdentifiers(data, self.tool_doi, 'hasPart')
         if self.descriptor_url is not None:
-            self.addHasPart(data, self.descriptor_url)
+            self.addRelatedIdentifiers(data, self.descriptor_url, 'hasPart')
         return data
 
-    # Check if value is a string
-    # try/except is needed for Python2/3 compatibility
-    def is_str(self, value):
-        try:
-            basestring
-        except NameError:
-            return isinstance(value, str)
-        return isinstance(value, basestring)
-
-    def addHasPart(self, data, identifier):
+    def addRelatedIdentifiers(self, data, identifier, relation):
         if data['metadata'].get('related_identifiers') is None:
             data['metadata']['related_identifiers'] = []
         data['metadata']['related_identifiers'].append({
             'identifier': identifier,
-            'relation': 'hasPart'
+            'relation': relation
         })
diff --git a/tools/python/boutiques/puller.py b/tools/python/boutiques/puller.py
@@ -62,7 +62,6 @@ def pull(self):
             searcher = Searcher(entry["zid"], self.verbose, self.sandbox,
                                 exact_match=True)
             r = searcher.zenodo_search()
-
             if not len(r.json()["hits"]["hits"]):
                 raise_error(ZenodoError, "Descriptor \"{0}\" "
                             "not found".format(entry["zid"]))

diff --git a/tools/python/boutiques/schema/descriptor.schema.json b/tools/python/boutiques/schema/descriptor.schema.json
@@ -22,6 +22,12 @@
             "description": "Tool description.",
             "type": "string"
         },
+        "deprecated-by-doi": {
+            "id": "http://github.com/boutiques/boutiques-schema/deprecated-by-doi",
+            "minLength": 1,
+            "description": "doi of the tool that deprecates the current one. May be set to 'true' if the current tool is deprecated but no specific tool deprecates it.",
+            "type": ["string", "boolean"]
+        },
         "author": {
             "id": "http://github.com/boutiques/boutiques-schema/author",
             "minLength": 1,

diff --git a/tools/python/boutiques/searcher.py b/tools/python/boutiques/searcher.py
@@ -7,13 +7,7 @@
 from operator import itemgetter
 from boutiques.logger import raise_error, print_info
 from boutiques.publisher import ZenodoError
-
-try:
-    # Python 3
-    from urllib.parse import quote
-except ImportError:
-    # Python 2
-    from urllib import quote
+from urllib.parse import quote
 
 
 class Searcher():
@@ -66,14 +60,25 @@ def __init__(self, query, verbose=False, sandbox=False, max_results=None,
 
     def search(self):
         results = self.zenodo_search()
-        num_results = len(results.json()["hits"]["hits"])
         total_results = results.json()["hits"]["total"]
-        print_info("Showing %d of %d results."
+        total_deprecated = len([h['metadata']['keywords'] for h in
+                                results.json()['hits']['hits'] if
+                                'metadata' in h and
+                                'keywords' in h['metadata'] and
+                                'deprecated' in h['metadata']['keywords']])
+        results_list = self.create_results_list_verbose(results.json()) if\
+            self.verbose else\
+            self.create_results_list(results.json())
+        num_results = len(results_list)
+        print_info("Showing %d of %d result(s)%s"
                    % (num_results if num_results < self.max_results
-                      else self.max_results, total_results))
-        if self.verbose:
-            return self.create_results_list_verbose(results.json())
-        return self.create_results_list(results.json())
+                      else self.max_results,
+                      total_results if self.verbose
+                      else total_results - total_deprecated,
+                      "." if self.verbose
+                      else ", exluding %d deprecated result(s)."
+                      % total_deprecated))
+        return results_list
 
     def zenodo_search(self):
         # Get all results
@@ -93,6 +98,10 @@ def create_results_list(self, results):
         results_list = []
         for hit in results["hits"]["hits"]:
             (id, title, description, downloads) = self.parse_basic_info(hit)
+            # skip hit if result is deprecated
+            keyword_data = self.get_keyword_data(hit["metadata"]["keywords"])
+            if 'deprecated' in keyword_data['other']:
+                continue
             result_dict = OrderedDict([("ID", id), ("TITLE", title),
                                       ("DESCRIPTION", description),
                                       ("DOWNLOADS", downloads)])
@@ -118,6 +127,8 @@ def create_results_list_verbose(self, results):
             result_dict = OrderedDict([("ID", id),
                                       ("TITLE", title),
                                       ("DESCRIPTION", description),
+                                      ("DEPRECATED", 'deprecated' in
+                                       keyword_data['other']),
                                       ("DOWNLOADS", downloads),
                                       ("AUTHOR", author),
                                       ("VERSION", version),