Display songs

generator/admin.py

@@ -118,7 +118,11 @@ def log_link(self, obj):
         match = re.search('([^" \']+\.log)', msg)
         if(match):
             url = settings.MEDIA_URL +'PDF/' + match.group(0)
-            return mark_safe("<a href='%s'>Voir les logs</a>" % url)
+            texurl = url[:-3] + "tex"
+            return mark_safe(
+                "Fichier <a href='%s'>.log</a>, <a href='%s'>.tex</a>"
+                % (url, texurl)
+                )
         return None
 
     log_link.allow_tags = True

generator/build.py

@@ -17,8 +17,7 @@
 from generator.models import Songbook, Layout, Task
 from django.conf import settings
 
-from patacrep.build import SongbookBuilder
-from patacrep.errors import SongbookError
+from generator.patacrep import build_songbook
 
 import os
 import hashlib
@@ -45,8 +44,8 @@ def __str__(self):
         return "[PDF Generator error] {0}". format(self.message)
 
 
-def generate_songbook(songbook, layout):
-    """Generate a PDF file by combining a songbook and a layout"""
+def prepare_songbook(songbook, layout):
+    """Return the filename and songbook content"""
 
     content = {}
     content.update(songbook.get_as_json())
@@ -57,18 +56,17 @@ def generate_songbook(songbook, layout):
     tmpfile = str(songbook.id) + '-' + str(layout.id) + '-' + \
               hashlib.sha1(str(content).encode()).hexdigest()[0:20]
 
+    return tmpfile, content
+
+def generate_songbook(filename, content):
+    """Generate a PDF file from the content"""
     try:
         os.chdir(SONGBOOKS_PDFS)
     except OSError:
         os.makedirs(SONGBOOKS_PDFS)
         os.chdir(SONGBOOKS_PDFS)
 
-    builder = SongbookBuilder(content, tmpfile)
-
-    for step in ["tex", "pdf", "sbx", "pdf", "clean"]:
-        try:
-            builder.build_steps([step])
-        except SongbookError as e:
-            raise GeneratorError("Error during the step '{0}': {1}".format(step, e))
+    steps = ["tex", "pdf", "sbx", "pdf", "clean"]
+    build_songbook(content, filename, steps)
 
-    return tmpfile + ".pdf"
+    return filename + ".pdf"

generator/songs.py → generator/management/commands/cleancatalog.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-#    Copyright (C) 2014 The Patacrep Team
+#    Copyright (C) 2015 The Patacrep Team
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU Affero General Public License as published by
@@ -14,12 +14,14 @@
 #    You should have received a copy of the GNU Affero General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""
-Functions for song file (.sg) rendering.
-"""
+from django.core.management.base import BaseCommand
 
-def parse_song(filename):
-    """Parse song 'filename', and return the corresponding HTML code."""
-    # TODO
-    with open(filename) as fd:
-        return fd.read()
+from generator.models import Artist, Songbook
+
+class Command(BaseCommand):
+    args = ""
+    help = "Remove all songs, artists and songbooks from the db"
+
+    def handle(self, *args, **options):
+        Artist.objects.all().delete()
+        Songbook.objects.all().delete()

generator/management/commands/importsongs.py

@@ -16,34 +16,38 @@
 
 from django.core.management.base import BaseCommand
 from django.conf import settings
-from django.db import transaction
 
-from generator.management.songs import import_song
+from generator.patacrep import extract_metadata
+from generator.management.songs import import_songs
 
 import os
+from pathlib import PurePosixPath
+import logging
 
+LOGGER = logging.getLogger(__name__)
 SONGS_DIR = os.path.join(settings.SONGS_LIBRARY_DIR, "songs")
 
-@transaction.atomic
 class Command(BaseCommand):
     args = ""
     help = "Import song information into db"
 
     def handle(self, *args, **options):
+        metadata = []
         for root, _dirs, filenames in os.walk(SONGS_DIR,
                                              topdown=True,
                                              onerror=_file_error,
                                              followlinks=False):
             for filename in filenames:
-                if filename.lower().endswith(".sg"):
-                    filepath = os.path.realpath(os.path.join(root, filename))
+                if filename.lower().endswith(".sgc"):
+                    fullpath = os.path.join(root, filename)
+                    filepath = str(PurePosixPath(fullpath).relative_to(SONGS_DIR))
                     try:
-                        import_song(filepath, SONGS_DIR)
+                        metadata.append(extract_metadata(filepath, settings.SONGS_LIBRARY_DIR))
                     except IOError as e:
                         self.stderr.write("*** Failed processing file : "
-                                          + filepath)
+                                          + fullpath)
                         self.stderr.write("    I/O error({0}): {1}"
                                           .format(e.errno, e.strerror))
-
+        import_songs(metadata)
 def _file_error(error):
     print(error)

generator/management/songs.py

@@ -16,87 +16,114 @@
 """
 Songs management utilities
 """
-from django.conf.global_settings import LANGUAGES
 from django.utils.text import slugify
+from django.db import transaction
 
 from generator.models import Song, Artist
-from patacrep.latex import parse_song
 
 import pygit2 as git
-import pprint
-import os
 import logging
+
 LOGGER = logging.getLogger(__name__)
 
-def import_song(filepath, song_directory):
-    '''Import a song in the database'''
-    with open(filepath) as song:
-        data = parse_song(song.read(), filepath)
-    LOGGER.info("Processing " +
-                pprint.pformat(data['@titles'][0]))
+@transaction.atomic
+def import_songs(metadata):
+    """Import all the metadata into the database"""
+    catalog = Catalog()
+    songs = []
+    for song in metadata:
+        title = song['titles'][0]
+        file_path = song['filepath']
+
+        if file_path in catalog.filepaths:
+            LOGGER.info("Skip: " + title
+                + "\t\t\t\t\t- Already in Database")
+            continue
+
+        slug = catalog.get_song_slug(title)
+        artist_id = catalog.get_artist_id(song['authors'][0])
+        language = song['lang']
+        object_hash = git.hashfile(song['fullpath'])
+
+        data = {}
+        for i in (
+            'title', 'file_path',
+            'slug', 'artist_id',
+            'language', 'object_hash',
+            ):
+            data[i] = locals()[i]
+
+        model = Song(**data)
+        LOGGER.info("Add: " + title)
+        songs.append(model)
+
+    Song.objects.bulk_create(songs)
+    LOGGER.info(
+        " -> " +
+        str(len(songs)) + 
+        " songs added to the db")
+
+class Catalog:
+    """Class used to efficiently compute slugs and prevent database collision"""
+    def __init__(self):
+        self._init_songs()
+        self._init_artists()
+
+    def _init_songs(self):
+        songs = Song.objects.values_list('file_path', 'slug')
+        if not songs:
+            self.filepaths = set()
+            self.song_slugs = set()
+        else:
+            self.filepaths, self.song_slugs = map(set, zip(*songs))
+
+    def get_song_slug(self, title):
+        slug = slugify(title)
+        if slug in self.song_slugs:
+            i = 1
+            tmp_slug = slug + '-' + str(i)
+            while tmp_slug in self.song_slugs:
+                i += 1
+                tmp_slug = slug + '-' + str(i)
+
+            LOGGER.info(
+                "SlugCollision: "
+                + slug + ". Use "
+                + tmp_slug + " instead for " + title)
+            slug = tmp_slug
+
+        self.song_slugs.add(slug)
+        return slug
+
+    def _init_artists(self):
+        artists = Artist.objects.values_list('id', 'slug', 'name')
+        self.artists = { slug: (id, name) for id, slug, name in artists }
+
+    def get_artist_name(self, name_tuple):
+        last_name, first_name = name_tuple
+        if first_name:
+            artist_name = first_name + ' ' + last_name
+        else:
+            artist_name = last_name
+
+        return artist_name
 
-    artist_name = data['by']
-    artist_slug = slugify(artist_name)
+    def get_artist_id(self, name_tuple):
+        artist_name = self.get_artist_name(name_tuple)
+        artist_slug = slugify(artist_name)
 
-    artist_model, created = Artist.objects.get_or_create(
+        try:
+            id, name = self.artists[artist_slug]
+            if (name != artist_name):
+                LOGGER.warning(
+                    "*** Artist name differs though "
+                    "slugs are equal : "
+                    + artist_name + " / " + name)
+            return id
+        except KeyError:
+            model = Artist.objects.create(
                             slug=artist_slug,
-                            defaults={'name': artist_name}
+                            name=artist_name,
                             )
-    if not created:
-        if (artist_model.name != artist_name):
-            LOGGER.warning(
-                "*** Artist name differs though "
-                "slugs are equal : "
-                + artist_name + " / "
-                + artist_model.name)
-
-    if (len(data['@languages']) > 1):
-        LOGGER.warning("*** Multiple languages "
-                        "in song file; we though"
-                        " only support one. "
-                        "Picking any.")
-    if (len(data['@languages']) > 0):
-        language_name = data["@languages"].pop()
-        language_code = next(
-                    (x for x in LANGUAGES
-                     if x[1].lower() == language_name.lower()
-                     ),
-                    ('', '')
-                     )[0]
-        if language_code == '':
-            LOGGER.warning("*** No code found for "
-                    "language : '" + language_name + "'")
-
-    song_title = data['@titles'][0]
-    song_slug = slugify(song_title)
-
-    object_hash = git.hashfile(filepath)
-    filepath_rel = os.path.relpath(filepath, song_directory)
-
-    import random
-
-    # For some reason - probably after having interrupted
-    # the generation - insertion fails because slug is
-    # empty, and there is already an empty one.
-    # We assign here a random value, that gets overwritten
-    # afterwards.
-    song_model, created = Song.objects.get_or_create(
-                            title=song_title,
-                            artist=artist_model,
-                            defaults={
-                                'title': song_title,
-                                'language': language_code,
-                                'file_path': filepath_rel,
-                                'slug': ('%06x' % random.randrange(16**6))
-                            })
-    if created:
-        if Song.objects.filter(slug=song_slug).exists():
-            song_slug += '-' + str(song_model.id)
-        song_model.slug = song_slug
-
-    else:
-        LOGGER.info("-> Already exists.")
-
-    artist_model.save()
-    song_model.object_hash = object_hash
-    song_model.save()
+            self.artists[artist_slug] = model.id, artist_name
+            return model.id