Ugarit Manifest Maker
Check-in [65f90322e5]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:OGG metadata parsing, and -Dfoo="bar" for defaults.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 65f90322e5274b62cc6729c73ae2fc53012752ab
User & Date: alaric 2015-05-28 11:23:10
Context
2015-06-12
19:50
Sorted directories in the manifest maker, fixed a typo in a property name, and moved to using "mtime" rather than "dc:modified", to reflect that it is just a POSIX filesystem attribute rather than anything relating to the actual content, and keeping compatability as such with existing Ugarit conventions. check-in: 6caa1bd9ec user: alaric tags: trunk
2015-05-28
11:23
OGG metadata parsing, and -Dfoo="bar" for defaults. check-in: 65f90322e5 user: alaric tags: trunk
2015-05-09
14:44
Moved to Dublin Core metadata where possible. check-in: 7eb028a964 user: alaric tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ugarit-manifest-maker.scm.

1
2
3

4
5
6
7
8
9
10
(use posix)
(use srfi-1)


(use ugarit-mime)
(use srfi-37)
(use miscmacros)
(use fnmatch)
(use exif)

(define *excludes* '())



>







1
2
3
4
5
6
7
8
9
10
11
(use posix)
(use srfi-1)

(use numbers)
(use ugarit-mime)
(use srfi-37)
(use miscmacros)
(use fnmatch)
(use exif)

(define *excludes* '())
42
43
44
45
46
47
48
49

50
51
52
53
54
55
56
    (dc:subject . #f)))

(define debug (lambda x (void)))
#;(define debug printf)

(define (alist-update! alist key value)
  (let ((pair (assq key alist)))
    (if (and (pair? pair) (not (cdr pair)))

        (begin
          (set-cdr! pair value) ;; Exists but is #f, so overwrite
          alist)
        (cons (cons key value) alist))))

(define genres
  '((0 . "Blues")







|
>







43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
    (dc:subject . #f)))

(define debug (lambda x (void)))
#;(define debug printf)

(define (alist-update! alist key value)
  (let ((pair (assq key alist)))
    (if (and (pair? pair)
             (not (cdr pair)))
        (begin
          (set-cdr! pair value) ;; Exists but is #f, so overwrite
          alist)
        (cons (cons key value) alist))))

(define genres
  '((0 . "Blues")
197
198
199
200
201
202
203

























































204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220

221
222
223
224
225
226
227
228
229
230
231
      str
      (if (char=? (string-ref str
                              (- (string-length str) 1))
                  #\x00)
          (substring str 0 (- (string-length str) 1))
          str)))


























































(define (get-mp3-metadata path)
  ;; FIXME http://id3.org/id3v2-00 / http://id3.org/id3v2.3.0
  (with-input-from-file path
    (lambda ()
      (let/cc
       abort*
       (let* ((metadata
               '((creator . #f)
                 (creation-date . #f)
                 (featuring . #f)
                 (collection-name . #f)
                 (collection-volume . #f)
                 (collection-volumes . #f)
                 (volume-index . #f)
                 (volume-size . #f)))
              (abort
               (lambda ()

                 (abort* metadata)))
              (read-id3v22
               (lambda ()
                 (debug "Reading v2.2")
                 (let* ((flags (read-byte))
                        (unsync (not (zero? (bitwise-and flags 128))))
                        (compressed (not (zero? (bitwise-and flags 64))))
                        (size4 (read-byte))
                        (size3 (read-byte))
                        (size2 (read-byte))
                        (size1 (read-byte))







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

|















>



|







199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
      str
      (if (char=? (string-ref str
                              (- (string-length str) 1))
                  #\x00)
          (substring str 0 (- (string-length str) 1))
          str)))

(define (read-utf16-be bytes rb)
  (let ((str (make-string (/ bytes 2))))
    (dotimes (idx (/ bytes 2))
             (let* ((ch1 (rb))
                    (ch2 (rb)))
               (string-set! str idx
                            (integer->char
                             (+ (* ch1 256)
                                ch2)))))
    str))

(define (read-utf16-le bytes rb)
  (let ((str (make-string (/ bytes 2))))
    (dotimes (idx (/ bytes 2))
             (let* ((ch2 (rb))
                    (ch1 (rb)))
               (string-set! str idx
                            (integer->char
                             (+ (* ch1 256)
                                ch2)))))
    str))

(define (load-id3-string encoding bytes rb)
  (case encoding
   ((0)
    (let ((str (make-string bytes)))
      (dotimes (idx bytes)
               (string-set! str idx (integer->char (rb))))
      str))
   ((1)
    ;; Look for a BOM
    (if (or (< bytes 2)
            (not (zero? (remainder bytes 2))))
        (let ((str (make-string bytes)))
          (dotimes (idx bytes)
                   (string-set! str idx (integer->char (rb))))
          (sprintf "Invalid UTF16 [~S]" str))
        (begin
          (let* ((ch1 (rb))
                 (ch2 (rb)))
            (cond
             ((and (= ch1 #xff)
                   (= ch2 #xfe))
              (read-utf16-le (- bytes 2) rb))
             ((and (= ch1 #xfe)
                   (= ch2 #xff))
              (read-utf16-be (- bytes 2) rb))
             (else
              (string-append
               (make-string 1 (integer->char (+ (* 256 ch1) ch2)))
               (read-utf16-be (- bytes 2) rb))))))))
   (else
    (let ((str (make-string bytes)))
      (dotimes (idx bytes)
               (string-set! str idx (integer->char (rb))))
      (sprintf "Unknown encoding [~S] [~S]" encoding str)))))

(define (get-mp3-metadata path)
  ;; http://id3.org/id3v2-00 / http://id3.org/id3v2.3.0
  (with-input-from-file path
    (lambda ()
      (let/cc
       abort*
       (let* ((metadata
               '((creator . #f)
                 (creation-date . #f)
                 (featuring . #f)
                 (collection-name . #f)
                 (collection-volume . #f)
                 (collection-volumes . #f)
                 (volume-index . #f)
                 (volume-size . #f)))
              (abort
               (lambda ()
                 (printf "   ;; Could not read metadata\n")
                 (abort* metadata)))
              (read-id3v22
               (lambda ()
                 (debug "Reading v2.2\n")
                 (let* ((flags (read-byte))
                        (unsync (not (zero? (bitwise-and flags 128))))
                        (compressed (not (zero? (bitwise-and flags 64))))
                        (size4 (read-byte))
                        (size3 (read-byte))
                        (size2 (read-byte))
                        (size1 (read-byte))
275
276
277
278
279
280
281
282






283
284
285
286
287
288
289
290
291
292
293
294
295
296
                            (fsize3 (rb))
                            (fsize2 (rb))
                            (fsize1 (rb))
                            (fsize (+
                                    fsize1
                                    (* 256 fsize2)
                                    (* 65536 fsize3)))
                            (encoding (rb))






                            (data
                             (remove-trailing-term-if-present
                              (list->string
                               (reverse
                                (list-tabulate
                                 (- fsize 1) ;; Remove encoding byte
                                 (lambda (i) (integer->char (rb)))))))))
                       (debug "FID: ~S size: ~S [~S]\n" fid fsize data)
                       (cond
                        ((string=? fid "\x00\x00\x00")
                         (abort* md))
                        ((string=? fid "TT2")
                         (loop (alist-update! md 'dc:title data)))
                        ((string=? fid "TP1")







|
>
>
>
>
>
>


<
<
<
|
<







335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350



351

352
353
354
355
356
357
358
                            (fsize3 (rb))
                            (fsize2 (rb))
                            (fsize1 (rb))
                            (fsize (+
                                    fsize1
                                    (* 256 fsize2)
                                    (* 65536 fsize3)))
                            (encoding (if (char=? (integer->char fid1) #\T)
                                          (rb)
                                          0))
                            (remaining-size
                             (if (char=? (integer->char fid1) #\T)
                                 (- fsize 1)
                                 fsize))
                            (data
                             (remove-trailing-term-if-present



                              (load-id3-string encoding remaining-size rb))))

                       (debug "FID: ~S size: ~S [~S]\n" fid fsize data)
                       (cond
                        ((string=? fid "\x00\x00\x00")
                         (abort* md))
                        ((string=? fid "TT2")
                         (loop (alist-update! md 'dc:title data)))
                        ((string=? fid "TP1")
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
                                 fid
                                 (string-append
                                  (make-string 1 (integer->char encoding))
                                  data))
                         (loop md))))))))
              (read-id3v23
               (lambda ()
                 (debug "Reading v2.3")
                 (let* ((flags (read-byte))
                        (unsync (not (zero? (bitwise-and flags 128))))
                        (extended (not (zero? (bitwise-and flags 64))))
                        (experimental (not (zero? (bitwise-and flags 32))))
                        (size4 (read-byte))
                        (size3 (read-byte))
                        (size2 (read-byte))







|







388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
                                 fid
                                 (string-append
                                  (make-string 1 (integer->char encoding))
                                  data))
                         (loop md))))))))
              (read-id3v23
               (lambda ()
                 (debug "Reading v2.3\n")
                 (let* ((flags (read-byte))
                        (unsync (not (zero? (bitwise-and flags 128))))
                        (extended (not (zero? (bitwise-and flags 64))))
                        (experimental (not (zero? (bitwise-and flags 32))))
                        (size4 (read-byte))
                        (size3 (read-byte))
                        (size2 (read-byte))
410
411
412
413
414
415
416
417






418
419
420
421
422
423
424
425
426
427
428
429
430
431
                            (fsize (+
                                    fsize1
                                    (* 256 fsize2)
                                    (* 65536 fsize3)
                                    (* 16777216 fsize4)))
                            (flags1 (rb))
                            (flags2 (rb))
                            (encoding (rb))






                            (data
                             (remove-trailing-term-if-present
                              (list->string
                               (reverse
                                (list-tabulate
                                 (- fsize 1) ;; Remove encoding byte
                                 (lambda (i) (integer->char (rb)))))))))
                       (debug "FID: ~S size: ~S [~S]\n" fid fsize data)
                       (cond
                        ((string=? fid "\x00\x00\x00\x00")
                         (abort* md))
                        ((string=? fid "TIT2")
                         (loop (alist-update! md 'dc:title data)))
                        ((string=? fid "TPE1")







|
>
>
>
>
>
>


<
<
<
|
<







472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487



488

489
490
491
492
493
494
495
                            (fsize (+
                                    fsize1
                                    (* 256 fsize2)
                                    (* 65536 fsize3)
                                    (* 16777216 fsize4)))
                            (flags1 (rb))
                            (flags2 (rb))
                            (encoding (if (char=? (integer->char fid1) #\T)
                                          (rb)
                                          0))
                            (remaining-size
                             (if (char=? (integer->char fid1) #\T)
                                 (- fsize 1)
                                 fsize))
                            (data
                             (remove-trailing-term-if-present



                              (load-id3-string encoding remaining-size rb))))

                       (debug "FID: ~S size: ~S [~S]\n" fid fsize data)
                       (cond
                        ((string=? fid "\x00\x00\x00\x00")
                         (abort* md))
                        ((string=? fid "TIT2")
                         (loop (alist-update! md 'dc:title data)))
                        ((string=? fid "TPE1")
481
482
483
484
485
486
487














































































































488
489

490





491
492
493
494
495
496
497
498


























































499
500
501
502
503
504
505
              (abort))
        (let ((version1 (read-byte))
              (version2 (read-byte)))
          (case version1
            ((2) (read-id3v22))
            ((3) (read-id3v23))
            (else (abort)))))))))















































































































(define (get-ogg-metadata path)

  ;; FIXME http://xiph.org/vorbis/doc/v-comment.html





  '((creator . #f)
    (creation-date . #f)
    (featuring . #f)
    (collection-name . #f)
    (collection-volume . #f)
    (collection-volumes . #f)
    (volume-index . #f)
    (volume-size . #f)))



























































(define (get-pdf-metadata path)
  '((dc:creator . #f)
    (dc:subject . #f)
    (dc:description . #f)
    (dc:created . #f)
    (dc:identifier . #f)







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


>
|
>
>
>
>
>
|
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
              (abort))
        (let ((version1 (read-byte))
              (version2 (read-byte)))
          (case version1
            ((2) (read-id3v22))
            ((3) (read-id3v23))
            (else (abort)))))))))

(define (read-ogg-lacing)
  (let loop ((lacing 0)
             (bytes 1))
    (let ((byte (read-byte)))
      (if (= byte 255)
          (loop (+ lacing byte) (+ bytes 1))
          (values bytes (+ lacing byte))))))

(define (read-ogg-page abort)
  (when (not (= (read-byte) #x4f)) ;; OggS magic number
        (abort))
  (when (not (= (read-byte) #x67))
        (abort))
  (when (not (= (read-byte) #x67))
        (abort))
  (when (not (= (read-byte) #x53))
        (abort))
  (when (not (= (read-byte) #x00)) ;; Version
        (abort))
  (read-byte) ;; Header type
  (read-byte) ;; Granule position
  (read-byte)
  (read-byte)
  (read-byte)
  (read-byte)
  (read-byte)
  (read-byte)
  (read-byte)
  (read-byte) ;; Stream serial number
  (read-byte)
  (read-byte)
  (read-byte)
  (read-byte) ;; Page sequence number
  (read-byte)
  (read-byte)
  (read-byte)
  (read-byte) ;; Page checksum
  (read-byte)
  (read-byte)
  (read-byte)
  (let ((segments (read-byte)))
    (let ((segment-sizes
           (reverse
            (let loop ((segments-left segments)
                       (segment-sizes '()))
              (if (zero? segments-left)
                  segment-sizes
                  (receive (bytes lacing)
                           (read-ogg-lacing)
                           (loop (- segments-left bytes)
                                 (cons lacing segment-sizes))))))))
      (let loop ((ss segment-sizes)
                 (segments '()))
        (if (null? ss)
            (reverse segments)
            (let ((data (read-string (car ss))))
              (loop (cdr ss)
                    (cons data segments))))))))

(define (read-ogg-u32)
  (let* ((s1 (read-byte))
         (s2 (read-byte))
         (s3 (read-byte))
         (s4 (read-byte)))
    (+
     s1
     (* 256 s2)
     (* 65536 s3)
     (* 16777216 s4))))

(define (read-ogg-string)
  (let ((length (read-ogg-u32)))
    (read-string length)))

(define (parse-ogg-comment str)
  (let ((sep-pos (string-index str #\=)))
    (if sep-pos
        (cons (string-take str sep-pos)
              (string-drop str (+ sep-pos 1)))
        str)))

(define (parse-ogg-comments comments abort)
  (with-input-from-string comments
    (lambda ()
      (let ((type (read-byte)))
        (unless (= type 3)
                (abort)))
      (unless (= (read-byte) #x76) #; "vorbis"
              (abort))
      (unless (= (read-byte) #x6f)
              (abort))
      (unless (= (read-byte) #x72)
              (abort))
      (unless (= (read-byte) #x62)
              (abort))
      (unless (= (read-byte) #x69)
              (abort))
      (unless (= (read-byte) #x73)
              (abort))
      (let ((vendor (read-ogg-string)))
        (debug "Vendor: [~S]\n" vendor))
      (let loop ((comments-left (read-ogg-u32))
                 (comments '()))
        (if (zero? comments-left)
            comments
            (begin
              (loop (- comments-left 1)
                    (cons (parse-ogg-comment (read-ogg-string)) comments))))))))


(define (get-ogg-metadata path)
  ;; http://xiph.org/vorbis/doc/framing.html
  ;; http://xiph.org/vorbis/doc/v-comment.html
  (with-input-from-file path
    (lambda ()
      (let/cc
       abort*
       (let* ((metadata
               '((creator . #f)
                 (creation-date . #f)
                 (featuring . #f)
                 (collection-name . #f)
                 (collection-volume . #f)
                 (collection-volumes . #f)
                 (volume-index . #f)
                 (volume-size . #f)))
              (abort
               (lambda ()
                 (printf "   ;; Could not read metadata\n")
                 (abort* metadata))))
         ;; Skip header
         (read-ogg-page abort)
         ;; Read comment header
         (let* ((comments-page (read-ogg-page abort))
                (comments-packet (first comments-page))
                (comments (parse-ogg-comments comments-packet abort)))
           ;; Process comments
           (let loop ((md (alist-copy metadata))
                      (comments comments))
             (if (null? comments)
                 md
                 (let ((key (car (car comments)))
                       (value (cdr (car comments))))
                   (cond
                    ((string-ci= key "artist")
                     (loop (alist-update! md 'cd:creator value)
                           (cdr comments)))
                    ((string-ci= key "performer")
                     (loop (alist-update! md 'dc:contributor value)
                           (cdr comments)))
                    ((string-ci= key "producer")
                     (loop (alist-update! md 'dc:contributor value)
                           (cdr comments)))
                    ((string-ci= key "title")
                     (loop (alist-update! md 'dc:title value)
                           (cdr comments)))
                    ((string-ci= key "album")
                     (loop (alist-update! md 'set:title value)
                           (cdr comments)))
                    ((string-ci= key "tracknumber")
                     (loop (alist-update! md 'set:index value)
                           (cdr comments)))
                    ((string-ci= key "totaltracks")
                     (loop (alist-update! md 'set:size value)
                           (cdr comments)))
                    ((string-ci= key "discnumber")
                     (loop (alist-update! md 'superset:index value)
                           (cdr comments)))
                    ((string-ci= key "totaldiscs")
                     (loop (alist-update! md 'superset:size value)
                           (cdr comments)))
                    ((string-ci= key "date")
                     (loop (alist-update! md 'dc:created value)
                           (cdr comments)))
                    ((string-ci= key "originaldate")
                     (loop (alist-update! md 'dc:created value)
                           (cdr comments)))
                    ((string-ci= key "genre")
                     (loop (alist-update! md 'dc:subject value)
                           (cdr comments)))
                    (else
                     (printf "   ;; Unknown Ogg tag ~S=~S\n"
                             key value)
                     (loop md (cdr comments)))))))))))))

(define (get-pdf-metadata path)
  '((dc:creator . #f)
    (dc:subject . #f)
    (dc:description . #f)
    (dc:created . #f)
    (dc:identifier . #f)
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
                     (printf "  (~S = ~S)\n" (car item) (cdr item))
                     (printf "  #;(~S = \"\")\n" (car item)))))
              (mime-type (extension->mimetype (string-append "."
                                                             (or
                                                              (pathname-extension path)
                                                              "")))))
         (printf "(object ~S\n" path)
         (printf "  (filename = ~S)\n" path)
         (printf "  (dc:format = ~S)\n" mime-type)
         (for-each print-item *defaults*)

         (newline)

         (cond
          ((string=? mime-type "image/jpeg")
           (for-each print-item
                     (get-jpeg-metadata path)))







|

|







771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
                     (printf "  (~S = ~S)\n" (car item) (cdr item))
                     (printf "  #;(~S = \"\")\n" (car item)))))
              (mime-type (extension->mimetype (string-append "."
                                                             (or
                                                              (pathname-extension path)
                                                              "")))))
         (printf "(object ~S\n" path)
         (printf "  (filename = ~S)\n" (pathname-strip-directory path))
         (printf "  (dc:format = ~S)\n" mime-type)
         (for-each print-item *defines*)

         (newline)

         (cond
          ((string=? mime-type "image/jpeg")
           (for-each print-item
                     (get-jpeg-metadata path)))
563
564
565
566
567
568
569
570


571
572
573
574
575
576
577
          ((string=? mime-type "application/postscript")
           (for-each print-item
                     (get-ps-metadata path)))

          (else
           (unless *got-name*
                   (printf "  (dc:title = ~S)\n" name))
           (printf "  #;(description = \"\")\n")))



         (newline)

         (let ((stat (file-stat path)))
           (printf "  (dc:modified = ~S)\n" (vector-ref stat 8))
           (printf "  (ctime = ~S)\n" (vector-ref stat 7)))








|
>
>







801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
          ((string=? mime-type "application/postscript")
           (for-each print-item
                     (get-ps-metadata path)))

          (else
           (unless *got-name*
                   (printf "  (dc:title = ~S)\n" name))
           (printf "  #;(description = \"\")\n")
           (printf "  ;; No metadata extraction available for ~S files"
                   (pathname-extension path))))

         (newline)

         (let ((stat (file-stat path)))
           (printf "  (dc:modified = ~S)\n" (vector-ref stat 8))
           (printf "  (ctime = ~S)\n" (vector-ref stat 7)))

607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
       vals))
    (option
     '(#\D "define") #t #f
     (lambda (o n x vals)
       (let ((pos (string-index x #\=)))
         (unless pos
                 (usage))
         (let ((key (string-take x pos))
               (value (string-drop x (+ pos 1))))
          (push! (cons key value) *defines*)))
       vals)))
   (lambda (o n x vals)
     (usage))
   cons
   '()))








|
|







847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
       vals))
    (option
     '(#\D "define") #t #f
     (lambda (o n x vals)
       (let ((pos (string-index x #\=)))
         (unless pos
                 (usage))
         (let ((key (string->symbol (string-take x pos)))
               (value (with-input-from-string (string-drop x (+ pos 1)) read)))
          (push! (cons key value) *defines*)))
       vals)))
   (lambda (o n x vals)
     (usage))
   cons
   '()))