Update README

README.md

@@ -1,16 +1,12 @@
-# Playphrase
-Search for specific words in srt files and watch with mpv.  
+# PlayPhrase
+
+Search for specific words or phrases in subtitle files and watch video fragments with [mpv](https://mpv.io/).
+
 Inspired by [videogrep](http://lav.io/2014/06/videogrep-automatic-supercuts-with-python/) and [playphrase.me](http://playphrase.me/).
 
-# Download
+# Video
 
-See [https://github.com/kelciour/playphrase/releases](https://github.com/kelciour/playphrase/releases)
-
-# Requirements
-
-* Python 2.7
-* grep
-* mpv
+[![YouTube: PlayPhrase for Movies](http://i.imgur.com/QZ9QSiO.png)](http://youtu.be/ciMEY3moATU)
 
 # Usage
 
@@ -19,33 +15,85 @@ Run ```python playphrase.py -i <media_dir> _init_``` to generate txt files from
 After that use 
 ```python playphrase.py -i <media_dir> <phrase>```
 
-### Keyboard Shortcuts 
-Use ```Enter``` to move to the next clip or ```Shift + <``` and ```Shift + >``` to switch between clips, ```q``` to close player.
+Regular expressions can be used in search, for example, \b for word boundary.
 
-More info: [https://mpv.io/manual/master/#keyboard-control](https://mpv.io/manual/master/#keyboard-control)
+### Keyboard Shortcuts 
+
+Use ```Enter``` to move to the next clip or ```Shift + <``` and ```Shift + >``` to switch between clips, ```Ctrl + Left``` and ```Ctrl + Right``` to move to the prev/next subtitle, ```q``` to close video player.
+
+More info: [https://mpv.io/manual/stable/#keyboard-control](https://mpv.io/manual/master/#keyboard-control)
 
 ### Batch Scripts
 
-There's ```.bat``` (Windows) and ```.sh``` (Linux) files to simplify user input. First time before running edit them and update ```media_dir``` path.
-Use ```quit```, ```exit``` or ```q```, ```x``` to exit from batch script.
+There's ```videogrep.bat``` (Windows) and ```videogrep.sh``` (Linux) files to simplify user input. First time before running edit them and update ```media_dir``` path. Use ```quit```, ```exit``` or ```q```, ```x``` to exit from the batch script.
+
+Here's a quick demo how to set up and run ```videogrep.bat``` on Windows ([YouTube](https://youtu.be/kEkXZY4LFCY)).
+
+### Additional Options:
 
-### Additional options:
 * ```-ph, --phrases GAP_BETWEEN_PHRASES``` 
 move start time of the clip to the beginning of the current phrase. Value is optional (default=1.75 seconds)
 * ```-l, --limit``` 
-maximum duration of the phrase (default=30 seconds)
+maximum duration of the phrase (default=60 seconds)
 * ```-p, --padding``` 
 padding in seconds to add to the start and end of each clip (default=0.0 seconds)
 * ```-e, --ending``` 
 play only matching lines (or phrases)
 * ```-r, --randomize``` 
-randomize the clips
+randomize clips
 * ```-o, --output``` 
 name of the file in which output of \'grep\' command will be written
 * ```-d, --demo``` 
 only show grep results
+* ```-a, --audio```
+create audio fragments
+* ```-v, --video```
+create video fragments
+* ```-vs, --video-sub```
+create video fragments with hardcoded subtitles
+* ```-s, --subtitles```
+create subtitles for fragments
+* ```-m, --mpv-options OPTIONS```
+mpv player options
+
+### Optional Configuration Changes
+
+For example, you can modify [mpv.conf](https://mpv.io/manual/stable/#configuration-files)
+
+
+```
+autofit=900
+geometry=50%:50%
+```
+
+and [input.conf](https://mpv.io/manual/stable/#interactive-control)
+
+
+```
+ENTER playlist-next force
+```
+
+More info: [https://mpv.io/manual/](https://mpv.io/manual/)
+
+# Download
+
+See [https://github.com/kelciour/playphrase/releases](https://github.com/kelciour/playphrase/releases/latest)
+
+# Usage with AudioBooks
+
+It's possible to use audiobooks as media input. For that purpose there's ```audiogrep.bat``` and ```audiogrep.sh``` files to simplify user input. But you need to generate subtitles for every audio file. It can be done almost automatically using [aeneas](https://github.com/readbeyond/aeneas). Also [csplit](https://en.wikipedia.org/wiki/Csplit) can be used to split text of the book by chapters and [Pragmatic Segmenter](https://github.com/diasks2/pragmatic_segmenter) to split chapter's content by "sentences".
+
+Here's example video how it looks like (YouTube):
+
+[![YouTube: PlayPhrase for AudioBooks](http://i.imgur.com/gUFXeVI.png)](https://youtu.be/LEyRfy7TsnE)
+
+# Requirements
+
+* python 2.7
+* grep
+* mpv
+* ffmpeg
 
 # Note
 
 * playphrase requires the subtitle track and the video file to have the exact same name, up to the extension.
-* just re-run your command if there was an error with pipe

playphrase.py

@@ -7,6 +7,7 @@ import re
 import sys
 import subprocess
 import time
+import locale
 
 from collections import OrderedDict
 
@@ -31,7 +32,16 @@ def get_time_parts(time):
 def seconds_to_srt_time(time):
     return '%02d:%02d:%02d,%03d' % get_time_parts(time)
 
-def read_subtitles(content):
+def read_subtitles(file_path):
+    content = open(file_path, 'rU').read()
+    
+    if content[:3]=='\xef\xbb\xbf': # with bom
+        content = content[3:]
+    
+    ret_code, content = convert_to_unicode(content)
+    if ret_code == False:
+        sys.exit(1)
+
     subs = []
     
     content = re.sub('\n\n+', '\n\n', content)
@@ -42,7 +52,7 @@ def read_subtitles(content):
             
             sub_start = srt_time_to_seconds(sub_timecode[0].strip())
             sub_end = srt_time_to_seconds(sub_timecode[1].strip())
-            sub_content = " ".join(sub_chunks[2:]).replace("\t", " ")
+            sub_content = "\n".join(sub_chunks[2:]).replace("\t", " ")
             sub_content = re.sub(r"<[^>]+>", "", sub_content)
             sub_content = re.sub(r"  +", " ", sub_content)
             sub_content = sub_content.strip()
@@ -58,7 +68,7 @@ def convert_into_sentences(en_subs, limit):
     for sub in en_subs:
         sub_start = sub[0]
         sub_end = sub[1]
-        sub_content = sub[2]
+        sub_content = sub[2].replace('\n', ' ')
 
         if len(subs) > 0: 
             prev_sub_start = subs[-1][0]
@@ -69,6 +79,7 @@ def convert_into_sentences(en_subs, limit):
                 sub_content[0] != '-' and
                 sub_content[0] != '"' and
                 sub_content[0] != u'♪' and
+                sub_content[0].isupper() != True and
                 (prev_sub_content[-1] != '.' or (sub_content[0:3] == '...' or (prev_sub_content[-3:] == '...' and sub_content[0].islower()))) and 
                 prev_sub_content[-1] != '?' and
                 prev_sub_content[-1] != '!' and
@@ -86,15 +97,38 @@ def convert_into_sentences(en_subs, limit):
 
     return subs
 
+def filter_subtitles(subs, clip_start, clip_end):
+    subs_filtered = []
+
+    for idx in range(len(subs)):
+        sub_start = subs[idx][0]
+        sub_end = subs[idx][1]
+        sub_content = subs[idx][2]
+        
+        if sub_end > clip_start and sub_start < clip_end:
+            subs_filtered.append((sub_start - clip_start, sub_end - clip_start, sub_content))
+
+        if sub_start > clip_end:
+            break
+
+    return subs_filtered
+
 def write_subtitles(filename, subs):
     f = open(filename, 'w')
 
-    for idx in range(len(subs)):
-        f.write("(%s, %s)" % (seconds_to_srt_time(subs[idx][0]), seconds_to_srt_time(subs[idx][1])))
-        f.write("\t")
-        f.write(subs[idx][2].encode('utf-8'))
-        f.write("\n")
-    
+    if filename.endswith('.srt'):
+        for idx in range(len(subs)):
+            f.write(str(idx+1) + "\n")
+            f.write(seconds_to_srt_time(subs[idx][0]) + " --> " + seconds_to_srt_time(subs[idx][1]) + "\n")
+            f.write(subs[idx][2].encode('utf-8') + "\n")
+            f.write("\n")
+    else:
+        for idx in range(len(subs)):
+            f.write("(%s, %s)" % (seconds_to_srt_time(subs[idx][0]), seconds_to_srt_time(subs[idx][1])))
+            f.write("\t")
+            f.write(subs[idx][2].encode('utf-8'))
+            f.write("\n")
+        
     f.close()
 
 def update_mpv_player_cmd(cmd_options, mpv_options):
@@ -114,6 +148,75 @@ def update_mpv_player_cmd(cmd_options, mpv_options):
 
     return cmd
 
+def update_progress(progress, num, max_num):
+    width = 25
+    n = int(progress / 100.0 * width)
+    sys.stdout.write("\r %3d%% [%s%s%s] %d/%d" % (progress, "=" * n, ">", " " * (width - n), num, max_num))
+    sys.stdout.flush()
+
+def get_fragment_filename(phrase):
+    s = phrase.strip().replace(' ', '_')
+    s = s.replace('.*', '...')
+    max_filename_length = 30
+    if len(s) > max_filename_length:
+        s = s[:max_filename_length] + "..."
+    return re.sub(r'(?u)[^-\w\'\.]', '', s)
+
+def create_fragments(search_phrase, clips, export_mode):
+    idx = 1
+    
+    update_progress(0, 0, len(clips))
+    for video_file, clip_start, clip_end in clips:
+        fragment_filename = get_fragment_filename(search_phrase)
+
+        if len(clips) > 1:
+            fragment_filename += "_" + str(idx).zfill(3)
+
+        ss = clip_start
+        to = clip_end
+        t = to - ss
+
+        t_fade = 0.2
+        af = "afade=t=in:st=%s:d=%s,afade=t=out:st=%s:d=%s" % (0, t_fade, t - t_fade, t_fade)
+        video_encoding_mode = "ultrafast"
+
+        if export_mode["audio"]:
+            cmd = " ".join(["ffmpeg", "-y", "-ss", str(ss), "-i", '"' + video_file + '"', "-loglevel", "quiet", "-t", str(t), "-map", "0:a:0", "-af", af, '"' + fragment_filename + ".mp3" + '"'])
+            p = subprocess.Popen(cmd)
+            p.wait()
+
+        if export_mode["video"]:
+            cmd = " ".join(["ffmpeg", "-y", "-ss", str(ss), "-i", '"' + video_file + '"', "-strict", "-2", "-loglevel", "quiet", "-t", str(t), "-map", "0:v:0", "-map", "0:a:0", "-c:v", "libx264", "-preset", video_encoding_mode, "-c:a", "aac", "-ac", "2", "-af", af, '"' + fragment_filename + ".mp4" + '"'])
+            p = subprocess.Popen(cmd)
+            p.wait()
+
+        if export_mode["video-sub"]:
+            srt_style = "FontName=Arial,FontSize=22"
+
+            srt_filename = video_file[:-4] + ".srt"
+            if srt_filename[1] == ":": # Windows
+                srt_filename = srt_filename.replace("\\", "\\\\\\\\")
+                srt_filename = srt_filename.replace(":", "\\\\:")
+                srt_filename = srt_filename.replace(",", "\\\\\\,")
+                srt_filename = srt_filename.replace("'", "\\\\\\'")
+
+            vf = "\"" + "subtitles=" + srt_filename + ":force_style='" + srt_style + "',setpts=PTS-STARTPTS" + "\""
+            af = "afade=t=in:st=%s:d=%s,afade=t=out:st=%s:d=%s,asetpts=PTS-STARTPTS" % (ss, t_fade, to - t_fade, t_fade)
+
+            cmd = " ".join(["ffmpeg", "-y", "-ss", str(ss), "-i", '"' + video_file + '"', "-strict", "-2", "-loglevel", "quiet", "-t", str(t), "-map", "0:v:0", "-map", "0:a:0", "-c:v", "libx264", "-preset", video_encoding_mode, "-c:a", "aac", "-ac", "2", "-vf", vf, "-af", af, "-copyts", '"' + fragment_filename + ".sub.mp4" + '"'])
+            p = subprocess.Popen(cmd)
+            p.wait()
+
+        if export_mode["subtitles"]:
+            subtitles_filename = video_file.rsplit('.', 1)[0] + ".srt"
+            subs = read_subtitles(subtitles_filename)
+            subs = filter_subtitles(subs, clip_start, clip_end)
+            write_subtitles(fragment_filename + ".srt", subs)
+
+        update_progress(float(idx) / len(clips) * 100, idx, len(clips))
+        
+        idx += 1
+
 def play_clips(clips, ending_mode, mpv_options):
     if len(clips) != 0:
         clip_filename, clip_start, clip_end = clips[0]
@@ -127,7 +230,7 @@ def play_clips(clips, ending_mode, mpv_options):
         with open(pipe_name, 'w'): # create pipe
             pass
 
-        p = subprocess.Popen(cmd, shell=False) # start mpv player in idle mode
+        p = subprocess.Popen(cmd) # start mpv player in idle mode
         
         with open(pipe_name, "wb", 0) as f_pipe:
             for clip_filename, clip_start, clip_end in clips:
@@ -151,30 +254,39 @@ def play_clips(clips, ending_mode, mpv_options):
                         p.kill()
                     return
 
-def main(media_dir, search_phrase, phrase_mode, phrases_gap, padding, limit, output_file, ending_mode, randomize_mode, demo_mode, mpv_options):
-    cmd = " ".join(["grep", "-r", "-n", "-i", "--include", "\*.txt", "-P", '"' + search_phrase + '"', '"' + media_dir + '"'])
+def main(media_dir, search_phrase, phrase_mode, phrases_gap, padding, limit, output_file, ending_mode, randomize_mode, demo_mode, mpv_options, audio_mode, video_mode, video_with_sub_mode, subtitles_mode):
+    search_phrase = search_phrase.decode(locale.getpreferredencoding())
+    search_phrase_in_utf8_representation = repr(search_phrase.encode("UTF-8"))
+    search_phrase_in_grep = "\"(?s)\(\d\d:\d\d:\d\d,\d\d\d\, \d\d:\d\d:\d\d,\d\d\d\)\\t[^\\n]*" + search_phrase_in_utf8_representation[1:-1] + "[^\\n]*\""
+
+    cmd = " ".join(["grep", "-r", "-z", "-o", "-i", "--include", "\*\.txt", "-P", search_phrase_in_grep, '"' + media_dir + '"'])
     p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, bufsize=-1)
     output, error = p.communicate()
 
     if p.returncode == 0:
+        matches = output.rstrip("\x00").split("\x00")
+        
         if output_file != None:
             with open(output_file, 'w') as f_results:
-                f_results.write(output)
+                f_results.write("\n".join(matches))
 
-        matches = output.splitlines()
-        
         clips = []
         for match in matches:
             filename, line = match.split(".txt:", 1)
-            line_number, line = line.split(":", 1)
-            line_number = int(line_number)
             
-            sub_timing, sub_content = line.split("\t", 1)            
-            sub_start, sub_end = sub_timing.strip("()").split(", ")
-            
-            match_sub_start = srt_time_to_seconds(sub_start)
-            match_sub_end = srt_time_to_seconds(sub_end)
+            lines = line.splitlines()
 
+            def get_line_timings(line):
+                sub_timing, sub_content = line.split("\t", 1)            
+                sub_start, sub_end = sub_timing.strip("()").split(", ")
+                return (sub_start, sub_end)
+
+            sub_start, sub_end = get_line_timings(lines[0])
+            match_sub_start = srt_time_to_seconds(sub_start)
+            
+            sub_start, sub_end = get_line_timings(lines[-1])
+            match_sub_end = srt_time_to_seconds(sub_end)
+            
             phrase_start = match_sub_start
             phrase_end = match_sub_end
 
@@ -182,10 +294,18 @@ def main(media_dir, search_phrase, phrase_mode, phrases_gap, padding, limit, out
                 with open(filename + ".txt") as f_txt:
                     txt_lines = f_txt.read().splitlines()
 
-                    txt_line_start_idx = line_number - 1
-                    txt_line_end_idx = line_number - 1
+                    def find_line_number(lines, line):
+                        for idx, lines in enumerate(lines):
+                            if line in lines:
+                                return idx + 1
 
-                    for txt_line in reversed(txt_lines[:line_number - 1]):
+                    line_number_start = find_line_number(txt_lines, lines[0])
+                    line_number_end = find_line_number(txt_lines, lines[-1])
+
+                    txt_line_start_idx = line_number_start - 1
+                    txt_line_end_idx = line_number_end - 1
+
+                    for txt_line in reversed(txt_lines[:line_number_start - 1]):
                         sub_timing, sub_content = txt_line.split("\t", 1)            
                         sub_start, sub_end = sub_timing.strip("()").split(", ")
 
@@ -198,7 +318,7 @@ def main(media_dir, search_phrase, phrase_mode, phrases_gap, padding, limit, out
                         else:
                             break
 
-                    for txt_line in txt_lines[line_number:]:
+                    for txt_line in txt_lines[line_number_end:]:
                         sub_timing, sub_content = txt_line.split("\t", 1)            
                         sub_start, sub_end = sub_timing.strip("()").split(", ")
 
@@ -251,7 +371,9 @@ def main(media_dir, search_phrase, phrase_mode, phrases_gap, padding, limit, out
         if randomize_mode:
             random.shuffle(clips)
 
-        if not demo_mode:
+        if audio_mode or video_mode or video_with_sub_mode or subtitles_mode:
+            create_fragments(search_phrase, clips, {"audio": audio_mode, "video": video_mode, "video-sub": video_with_sub_mode, "subtitles": subtitles_mode})
+        elif not demo_mode:
             play_clips(clips, ending_mode, mpv_options)
 
     elif p.returncode == 1:
@@ -294,15 +416,7 @@ def init(media_dir, limit):
 
                 print file_path
 
-                with open(file_path, 'rU') as f_srt:
-                    content = f_srt.read()
-                    if content[:3]=='\xef\xbb\xbf': # with bom
-                        content = content[3:]
-                    ret_code, content = convert_to_unicode(content)
-                    if ret_code == False:
-                        sys.exit(1)
-
-                subs = read_subtitles(content)
+                subs = read_subtitles(file_path)
                 subs = convert_into_sentences(subs, limit)
 
                 write_subtitles(file_path[:-4] + ".txt", subs)
@@ -316,7 +430,7 @@ def parse_args(argv):
         print "Search phrase can't be empty"
         sys.exit()
 
-    args = {"padding": 0, "limit": 15, "output_file": None, "phrase_mode": False, "phrases_gap":1.75, "search_phrase":search_phrase, "ending_mode":False, "randomize_mode":False, "demo_mode":False, "mpv_options":""}
+    args = {"padding": 0, "limit": 60, "output_file": None, "phrase_mode": False, "phrases_gap":1.25, "search_phrase":search_phrase, "ending_mode":False, "randomize_mode":False, "demo_mode":False, "mpv_options":"", "audio_mode":False, "video_mode":False, "video_with_sub_mode":False, "subtitles_mode":False }
     
     argv = argv[:-1]
     idx = 0
@@ -347,6 +461,14 @@ def parse_args(argv):
             args["randomize_mode"] = True
         elif argv[idx] == "--demo" or argv[idx] == "-d":
             args["demo_mode"] = True
+        elif argv[idx] == "--audio" or argv[idx] == "-a":
+            args["audio_mode"] = True
+        elif argv[idx] == "--video" or argv[idx] == "-v":
+            args["video_mode"] = True
+        elif argv[idx] == "--video-sub" or argv[idx] == "-vs":
+            args["video_with_sub_mode"] = True
+        elif argv[idx] == "--subtitles" or argv[idx] == "-s":
+            args["subtitles_mode"] = True
         elif argv[idx] == "--phrases" or argv[idx] == "-ph":
             args["phrase_mode"] = True
             if idx + 1 < len(argv):
@@ -372,22 +494,28 @@ def parse_args(argv):
     return args
 
 def usage():
-    print "python videogrep.py -i <media_dir> <phrase>"
-    print "python videogrep.py -i <media_dir> _init_"
+    print "Usage: playphrase -i <media_dir> <phrase>"
+    print ""
+    print "Init: playphrase -i <media_dir> _init_"
     print ""
     print "Additional options:"
-    print "-ph, --phrases GAP_BETWEEN_PHRASES", "\t", "move start time of the clip to the beginning of the current phrase. Value is optional (default=1.75 seconds)"
-    print "-l, --limit", "\t", "maximum duration of the phrase (default=30 seconds)"
-    print "-p, --padding", "\t", "padding in seconds to add to the start and end of each clip (default=0.0 seconds)"
-    print "-e, --ending", "\t", "play only matching lines (or phrases)"
-    print "-r, --randomize", "\t", "randomize the clips"
-    print "-o, --output", "\t", "name of the file in which output of \'grep\' command will be written"
-    print "-d, --demo", "\t", "only show grep results"
-    print "-m, --mpv-options OPTIONS", "\t", "mpv player options"
+    print "-ph, --phrases GAP_BETWEEN_PHRASES", " ", "move start time of the clip to the beginning of the current phrase. Value is optional (default=1.25 seconds)"
+    print "-l, --limit", "      ", "maximum duration of the phrase (default=60 seconds)"
+    print "-p, --padding", "    ", "padding in seconds to add to the start and end of each clip (default=0.0 seconds)"
+    print "-e, --ending", "     ", "play only matching lines (or phrases)"
+    print "-r, --randomize", "  ", "randomize the clips"
+    print "-o, --output", "     ", "name of the file in which output of \'grep\' command will be written"
+    print "-d, --demo", "       ", "only show grep results"
+    print "-a, --audio", "      ", "create audio fragments"
+    print "-v, --video", "      ", "create video fragments"
+    print "-vs, --video-sub", " ", "create video fragments with hardcoded subtitles"
+    print "-s, --subtitles", "  ", "create subtitles for fragments"
+    print "-m, --mpv-options OPTIONS", " ", "mpv player options"
 
 if __name__ == '__main__':
     os.environ["PATH"] += os.pathsep + "." + os.sep + "utils" + os.sep + "grep"
     os.environ["PATH"] += os.pathsep + "." + os.sep + "utils" + os.sep + "mpv"
+    os.environ["PATH"] += os.pathsep + "." + os.sep + "utils" + os.sep + "ffmpeg"
 
     args = parse_args(sys.argv[1:])
     if args != False:
@@ -395,8 +523,8 @@ if __name__ == '__main__':
             init(args["media_dir"], args["limit"])
         else:
             if need_update(args["media_dir"]):
-                print "WARNING: number of '.srt' and '.txt' files doesn't match. Maybe you need to use 'videogrep <media_dir> _init_'."
+                print "WARNING: number of '.srt' and '.txt' files doesn't match. Maybe you need to use 'playphrase <media_dir> _init_'."
             
-            main(args["media_dir"], args["search_phrase"], args["phrase_mode"], args["phrases_gap"], args["padding"], args["limit"], args["output_file"], args["ending_mode"], args["randomize_mode"], args["demo_mode"], args["mpv_options"])
+            main(args["media_dir"], args["search_phrase"], args["phrase_mode"], args["phrases_gap"], args["padding"], args["limit"], args["output_file"], args["ending_mode"], args["randomize_mode"], args["demo_mode"], args["mpv_options"], args["audio_mode"], args["video_mode"], args["video_with_sub_mode"], args["subtitles_mode"])
     else:
         usage()