Revert "Get texidoc translations out of snippets source files"

This reverts commit 512d405d07aba8742658902b105a94ebbd40c2dc.

2 files changed, 133 insertions, 213 deletions

diff --git a/scripts/auxiliar/makelsr.py b/scripts/auxiliar/makelsr.py
index 61526963be..6927476be9 100755
--- a/scripts/auxiliar/makelsr.py
+++ b/scripts/auxiliar/makelsr.py
@@ -4,26 +4,22 @@ import sys
 import os
 import glob
 import re
-import optparse
-import tempfile
 
-lilypond_flags = "-dno-print-pages -dsafe"
-
-lys_from_lsr = os.path.join ('Documentation', 'snippets')
-new_lys = os.path.join ('Documentation', 'snippets', 'new')
-ly_output = os.path.join (tempfile.gettempdir (), 'lsrtest')
-
-# which convert-ly to use
-if os.path.isfile ("out/bin/convert-ly"):
-    conv_path = "out/bin/"
-elif os.path.isfile ("build/out/bin/convert-ly"):
-    conv_path = "build/out/bin/"
-else:
-    conv_path=''
-convert_ly = conv_path + 'convert-ly'
-lilypond_bin = conv_path + 'lilypond'
+sys.path.append ('python')
+import langdefs
 
+DEST = os.path.join ('Documentation', 'snippets')
+NEW_LYS = os.path.join ('Documentation', 'snippets', 'new')
+TEXIDOCS = [os.path.join ('Documentation', language_code, 'texidocs')
+            for language_code in langdefs.LANGDICT]
 
+USAGE = '''  Usage: makelsr.py [LSR_SNIPPETS_DIR]
+This script must be run from top of the source tree;
+it updates snippets %(DEST)s with snippets
+from %(NEW_LYS)s or LSR_SNIPPETS_DIR.
+If a snippet is present in both directories, the one
+from %(NEW_LYS)s is preferred.
+''' % vars ()
 
 LY_HEADER_LSR = '''%% DO NOT EDIT this file manually; it is automatically
 %% generated from LSR http://lsr.dsi.unimi.it
@@ -39,141 +35,38 @@ LY_HEADER_NEW = '''%% DO NOT EDIT this file manually; it is automatically
 %% and then run scripts/auxiliar/makelsr.py
 %%
 %% This file is in the public domain.
-''' % new_lys
-
-options_parser = optparse.OptionParser (
-    description = "makelsr - update snippets directory from LSR",
-    usage = '''%%prog [options] [LSR_SNIPPETS_DIR]
-Unless -s option is specified, this script must be run from top of the
-source tree. If LSR_SNIPPETS_DIR is not specified, it defaults to
-current directory.
-
-Remove snippets in TOP_SOURCE_DIR/%(lys_from_lsr)s and put in snippets
-from LSR_SNIPPETS_DIR run through convert-ly or from
-TOP_SOURCE_DIR/%(new_lys)s; if a snippet is present in both
-directories, the one from TOP_SOURCE_DIR/%(new_lys)s is preferred.
-All written snippets are copied in LY_OUTPUT
-with appending translations from .texidoc files and are tested with
-lilypond with flags %(lilypond_flags)s
-
-''' % vars ())
-
-options_parser.add_option ('-s', '--top-source',
-                           dest="top_source_dir",
-                           action="store",
-                           metavar="TOP_SOURCE_DIR",
-                           default=".",
-                           help="set LilyPond top source directory")
-
-options_parser.add_option ('-o', '--ly-output',
-                           dest="ly_output",
-                           action="store",
-                           metavar="LY_OUTPUT",
-                           default=ly_output,
-                           help="set LilyPond output files temporary directory")
-
-options_parser.add_option ('-p', '--path',
-                           dest="bin_path",
-                           action="store",
-                           metavar="LY_PATH",
-                           default="out/bin",
-                           help="directory where looking for LilyPond binaries")
-
-options_parser.add_option ('-c', '--convert-ly',
-                           dest="convert_ly",
-                           action="store",
-                           metavar="CONVERT-LY",
-                           default="LY_PATH/convert-ly",
-                           help="convert-ly binary to use")
-
-options_parser.add_option ('-l', '--lilypond-binary',
-                           dest="lilypond_bin",
-                           action="store",
-                           metavar="LILYPOND_BIN",
-                           default="LY_PATH/lilypond",
-                           help="lilypond binary to use")
-
-(options, args) = options_parser.parse_args ()
-
-if not os.path.isdir (options.top_source_dir):
-    sys.stderr.write ("Error: top source: %s: not a directory\n" % options.top_source_dir)
-    sys.exit (4)
-
-lys_from_lsr = os.path.normpath (os.path.join (options.top_source_dir, lys_from_lsr))
-new_lys = os.path.normpath (os.path.join (options.top_source_dir, new_lys))
-sys.path.append (os.path.normpath (os.path.join (options.top_source_dir, 'python')))
-import langdefs
-texidoc_dirs = [
-    os.path.normpath (os.path.join (options.top_source_dir, 'Documentation', language_code, 'texidocs'))
-    for language_code in langdefs.LANGDICT]
-
-if not os.path.isdir (lys_from_lsr):
-    sys.stderr.write ("Error: snippets path: %s: not a directory\n" % lys_from_lsr)
-    sys.exit (3)
-if not os.path.isdir (new_lys):
-    sys.stderr.write ("Error: new snippets path: %s: not a directory\n" % lys_from_lsr)
-    sys.exit (3)
-
-ly_output_ok = False
-if os.path.isdir (options.ly_output):
-    ly_output = options.ly_output
-    ly_output_ok = True
-elif os.path.exists (options.ly_output):
-    try:
-        os.unlink (options.ly_output)
-    except Exception as e:
-        sys.stderr.write ("Warning: could not delete file before creating directory: %s\n" % e)
-    else:
-        try:
-            os.makedirs (options.ly_output)
-        except Exception as e:
-            sys.stderr.write ("Warning: could not create directory: %s\n" % e)
-        else:
-            ly_output = options.ly_output
-            ly_output_ok = True
-else:
-    try:
-        os.makedirs (options.ly_output)
-    except Exception as e:
-        sys.stderr.write ("Warning: could not create directory: %s\n" % e)
-    else:
-        ly_output = options.ly_output
-        ly_output_ok = True
-if not ly_output_ok:
-    ly_output = tempfile.gettempdir ()
-    sys.stderr.write ("Warning: could not use or create directory %s, using default %s\n" % (options.ly_output, ly_output))
+''' % NEW_LYS
 
 def exit_with_usage (n=0):
-    options_parser.print_help (sys.stderr)
+    sys.stderr.write (USAGE)
     sys.exit (n)
 
-if len (args):
-    in_dir = args[0]
+TAGS = []
+
+if len (sys.argv) >= 2:
+    in_dir = sys.argv[1]
     if not (os.path.isdir (in_dir)):
-        sys.stderr.write ("Error: %s: not a directory\n" % in_dir)
-        sys.exit (4)
-    if len (args) > 1:
+        sys.stderr.write (in_dir + ' is not a directory.\n')
+        exit (2)
+    if len (sys.argv) >= 3:
         exit_with_usage (2)
+    if not (os.path.isdir (DEST) and os.path.isdir (NEW_LYS)):
+        exit_with_usage (3)
+    TAGS = os.listdir (in_dir)
 else:
-    in_dir = '.'
+    in_dir = ''
 
-if options.convert_ly == "LY_PATH/convert-ly":
-    convert_ly = os.path.join (options.bin_path, "convert-ly")
-else:
-    convert_ly = options.convert_ly
-if not os.path.isfile (convert_ly):
-    sys.stderr.write ("Warning: %s: no such file")
-    convert_ly = "convert-ly"
-if options.lilypond_bin == "LY_PATH/lilypond":
-    lilypond_bin = os.path.join (options.bin_path, "lilypond")
+# which convert-ly to use
+if os.path.isfile("out/bin/convert-ly"):
+    conv_path='out/bin/'
+elif os.path.isfile("build/out/bin/convert-ly"):
+    conv_path='build/out/bin/'
 else:
-    lilypond_bin = options.lilypond_bin
-if not os.path.isfile (lilypond_bin):
-    sys.stderr.write ("Warning: %s: no such file")
-    lilypond_bin = "lilypond"
-sys.stderr.write ("Using %s, %s\n" % (convert_ly, lilypond_bin))
+    conv_path=''
+convert_ly=conv_path+'convert-ly'
+lilypond_bin=conv_path+'lilypond'
 
-tags = os.listdir (in_dir)
+print 'using '+convert_ly
 
 unsafe = []
 unconverted = []
@@ -211,15 +104,32 @@ def add_version (ly_code):
     return '''%% Note: this file works from version ''' + \
         ly_new_version_re.search (ly_code).group (1) + '\n'
 
+s = 'Translation of GIT [Cc]ommittish'
+texidoc_chunk_re = re.compile (r'^(?:%+\s*' + s + \
+    r'.+)?\s*(?:texidoc|doctitle)([a-zA-Z]{2,4})\s+=(?:.|\n)*?(?=%+\s*' + \
+    s + r'|\n\} % begin verbatim|\n  (?:doctitle|texidoc|lsrtags) |$(?!.|\n))', re.M)
+
+def update_translated_texidoc (m, snippet_path, visited_languages):
+    base = os.path.splitext (os.path.basename (snippet_path))[0]
+    language_code = m.group (1)
+    visited_languages.append (language_code)
+    texidoc_path = os.path.join ('Documentation', language_code,
+                                 'texidocs', base + '.texidoc')
+    if os.path.isfile (texidoc_path):
+        return open (texidoc_path).read ()
+    else:
+        return m.group (0)
+
 def escape_backslashes_in_header(snippet):
     # ASSUME: the \header exists.
     header_char_number_start = snippet.find('\header {')
     header_char_number_end = snippet.find('} % begin verbatim')
 
     header = snippet[header_char_number_start:header_char_number_end]
-    # only one level of escaping happening here
-    # thanks to raw strings
-    new_header = re.sub(r"@code\{\\([a-zA-Z])", r"@code{\\\\\1", header)
+    # two levels of escaping happening here -- 4\ means 1\
+    # and the 10\ means two \ backslashes (that's 8\ ), and
+    # one backreference to group 1 (that's two 2\ ).
+    new_header = re.sub("@code\{\\\\([a-zA-Z])", "@code{\\\\\\\\\\1", header)
     escaped_snippet = (snippet[:header_char_number_start] +
         new_header + snippet[header_char_number_end:])
     return escaped_snippet
@@ -227,42 +137,50 @@ def escape_backslashes_in_header(snippet):
 def copy_ly (srcdir, name, tags):
     global unsafe
     global unconverted
-    dest = os.path.join (lys_from_lsr, name)
+    dest = os.path.join (DEST, name)
     tags = ', '.join (tags)
-    file_path = os.path.join (srcdir, name)
-    sys.stderr.write ("\nmakelsr.py: reading %s\n" % file_path)
-    s = open (file_path).read ()
+    s = open (os.path.join (srcdir, name)).read ()
+    sys.stderr.write ('\nmakelsr.py: reading ' + os.path.join (srcdir, name) + '\n')
+
+    for path in TEXIDOCS:
+        texidoc_translation_path = \
+            os.path.join (path, os.path.splitext (name)[0] + '.texidoc')
+        if os.path.exists (texidoc_translation_path):
+            texidoc_translation = open (texidoc_translation_path).read ()
+            # Since we want to insert the translations verbatim using a 
+            # regexp, \\ is understood as ONE escaped backslash. So we have
+            # to escape those backslashes once more...
+            texidoc_translation = texidoc_translation.replace ('\\', '\\\\')
+            s = begin_header_re.sub ('\\g<0>\n' + texidoc_translation, s, 1)
 
     s = doctitle_re.sub (doctitle_sub, s)
-    if "new" in srcdir:
-        s = LY_HEADER_NEW + add_version (s) + s
-    else:
+    if in_dir and in_dir in srcdir:
         s = LY_HEADER_LSR + add_tags (s, tags)
+    else:
+        s = LY_HEADER_NEW + add_version (s) + s
 
     s = mark_verbatim_section (s)
     s = lsr_comment_re.sub ('', s)
     s = strip_white_spaces_re.sub ('', s)
     s = escape_backslashes_in_header (s)
-    sys.stderr.write ("makelsr.py: writing %s\n" % dest)
     open (dest, 'w').write (s)
+    sys.stderr.write ('makelsr.py: writing ' + dest + '\n')
 
     e = os.system (convert_ly+(" -d -e '%s'" % dest))
     if e:
         unconverted.append (dest)
     if os.path.exists (dest + '~'):
         os.remove (dest + '~')
-    # no need to check snippets from Documentation/snippets/new
-    if not "new" in srcdir:
-        e = os.system (
-            "%s %s -o %s '%s'" %
-            (lilypond_bin, lilypond_flags, ly_output, dest))
+    # no need to check snippets from input/new
+    if in_dir and in_dir in srcdir:
+        e = os.system ("%s -dno-print-pages -dsafe -o /tmp/lsrtest '%s'" %(lilypond_bin, dest))
         if e:
             unsafe.append (dest)
 
 def read_source_with_dirs (src):
     s = {}
     l = {}
-    for tag in tags:
+    for tag in TAGS:
         srcdir = os.path.join (src, tag)
         l[tag] = set (map (os.path.basename,
                            glob.glob (os.path.join (srcdir, '*.ly'))))
@@ -278,14 +196,14 @@ tags_re = re.compile ('lsrtags\\s*=\\s*"(.+?)"')
 
 def read_source (src):
     s = {}
-    l = dict ([(tag, set()) for tag in tags])
+    l = dict ([(tag, set()) for tag in TAGS])
     for f in glob.glob (os.path.join (src, '*.ly')):
         basename = os.path.basename (f)
         m = tags_re.search (open (f, 'r').read ())
         if m:
             file_tags = [tag.strip() for tag in m.group (1). split(',')]
             s[basename] = (src, file_tags)
-            [l[tag].add (basename) for tag in file_tags if tag in tags]
+            [l[tag].add (basename) for tag in file_tags if tag in TAGS]
         else:
             notags_files.append (f)
     return s, l
@@ -301,23 +219,62 @@ def dump_file_list (file, file_list, update=False):
     f = open (file, 'w')
     f.write ('\n'.join (sorted (new_list)) + '\n')
 
-## clean out existing lys and generated files
-map (os.remove, glob.glob (os.path.join (lys_from_lsr, '*.ly')) +
-     glob.glob (os.path.join (lys_from_lsr, '*.snippet-list')))
-
-# read LSR source where tags are defined by subdirs
-snippets, tag_lists = read_source_with_dirs (in_dir)
-
-# read input/new where tags are directly defined
-s, l = read_source (new_lys)
-snippets.update (s)
-for t in tags:
-    tag_lists[t].update (l[t])
+def update_ly_in_place (snippet_path):
+    visited_languages = []
+    contents = open (snippet_path).read ()
+    contents = texidoc_chunk_re.sub \
+        (lambda m: update_translated_texidoc (m,
+                                              snippet_path,
+                                              visited_languages),
+         contents)
+    need_line_break_workaround = False
+    for language_code in langdefs.LANGDICT:
+        if not language_code in visited_languages:
+            base = os.path.splitext (os.path.basename (snippet_path))[0]
+            texidoc_path = os.path.join ('Documentation', language_code,
+                         'texidocs', base + '.texidoc')
+            if os.path.isfile (texidoc_path):
+                texidoc_translation = open (texidoc_path).read ()
+                texidoc_translation = texidoc_translation.replace ('\\', '\\\\')
+                contents = begin_header_re.sub ('\\g<0>\n' + texidoc_translation, contents, 1)
+        else:
+            need_line_break_workaround = True
+    contents = doctitle_re.sub (doctitle_sub, contents)
+    contents = escape_backslashes_in_header (contents)
+
+    # workaround for a bug in the regex's that I'm not smart
+    # enough to figure out.  -gp
+    if need_line_break_workaround:
+        first_translated = contents.find('%% Translation of')
+        keep = contents[:first_translated+5]
+        contents = keep + contents[first_translated+5:].replace('%% Translation of', '\n%% Translation of')
+
+    open (snippet_path, 'w').write (contents)
+
+if in_dir:
+    ## clean out existing lys and generated files
+    map (os.remove, glob.glob (os.path.join (DEST, '*.ly')) +
+         glob.glob (os.path.join (DEST, '*.snippet-list')))
+
+    # read LSR source where tags are defined by subdirs
+    snippets, tag_lists = read_source_with_dirs (in_dir)
+
+    # read input/new where tags are directly defined
+    s, l = read_source (NEW_LYS)
+    snippets.update (s)
+    for t in TAGS:
+        tag_lists[t].update (l[t])
+else:
+    snippets, tag_lists = read_source (NEW_LYS)
+    ## update texidocs of snippets that don't come from NEW_LYS
+    for snippet_path in glob.glob (os.path.join (DEST, '*.ly')):
+        if not os.path.basename (snippet_path) in snippets:
+            update_ly_in_place (snippet_path)
 
 for (name, (srcdir, tags)) in snippets.items ():
     copy_ly (srcdir, name, tags)
 for (tag, file_set) in tag_lists.items ():
-    dump_file_list (os.path.join (lys_from_lsr, tag + '.snippet-list'),
+    dump_file_list (os.path.join (DEST, tag + '.snippet-list'),
                     file_set, update=not(in_dir))
 if unconverted:
     sys.stderr.write ('These files could not be converted successfully by convert-ly:\n')
@@ -330,7 +287,7 @@ if unsafe:
     sys.stderr.write ('''
 
 Unsafe files printed in lsr-unsafe.txt: CHECK MANUALLY!
-  git add %(lys_from_lsr)s/*.ly
+  git add %s/*.ly
   xargs git diff HEAD < lsr-unsafe.txt
 
-''' % vars ())
+''' % DEST)
diff --git a/scripts/build/makesnippets.py b/scripts/build/makesnippets.py
deleted file mode 100644
index 9e7c5a63fe..0000000000
--- a/scripts/build/makesnippets.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#!@PYTHON@
-# makesnippets.py
-
-'''USAGE: makesnippets.py INPUT_DIR OUTPUT_DIR DOC_DIR
-
-Read all .ly files from INPUT_DIR, insert translations from .texidoc
-files found in DOC_DIR/LANG/texdiocs, and write ther result to OUTPUT_DIR.'''
-
-import glob
-import sys
-import os.path
-import re
-
-import langdefs
-
-(input_dir, output_dir, doc_dir) = sys.argv[1:4]
-
-texidoc_dirs = [os.path.join (doc_dir, language_code, 'texidocs')
-                for language_code in langdefs.LANGDICT]
-
-begin_header_re = re.compile (r'\\header\s*{', re.M)
-
-for f in glob.glob (os.path.join (input_dir, '*.ly')):
-    name = os.path.basename (f)
-    s = open (f).read ()
-    for path in texidoc_dirs:
-        texidoc_translation_path = \
-            os.path.join (path, os.path.splitext (name)[0] + '.texidoc')
-        if os.path.exists (texidoc_translation_path):
-            texidoc_translation = open (texidoc_translation_path).read ()
-            # Since we want to insert the translations verbatim using a
-            # regexp, \\ is understood as ONE escaped backslash. So we have
-            # to escape those backslashes once more...
-            texidoc_translation = texidoc_translation.replace ('\\', '\\\\')
-            s = begin_header_re.sub ('\\g<0>\n' + texidoc_translation, s, 1)
-    dest = os.path.join (output_dir, name)
-    open (dest, 'w').write (s)