Initial commit

README.md

@@ -0,0 +1,93 @@
+# Simple photo cataloguer
+
+Just copy/hardlink photos (or video, or any other files) from one place to
+another, separating them in sub-directories like `$ROOT/year/month/day/`.
+
+### TL;DR 
+
+I have a smartphone, I have a Syncthing ~~uugh... SmartThing~~ and all photos
+from smartphone nicely synced to my PC without my attention. But I can't just
+keep all photos in synced folder: if I'll clean my phone memory - all photos 
+from pc will be cleaned too. I need to not forget copy files in another 
+place before cleaning phone's memory. Also, I can't  just drop all photos in 
+one dir - I will not find anything there later, and a folder with thousands 
+photos looks like a bad idea from either side.    
+So I create this tool in one evening. All it does - copy (or create hardlinks)
+for files from one place to another, creating basic date-aware directories
+structure for that files.
+
+
+## Installing
+```bash
+go install github.com/derfenix/photocatalog
+```
+Optionally you could copy created binary from the GO's bin path to 
+system or user $PATH, e.g. /usr/local/bin/.
+```bash
+sudo cp ${GOPATH}/bin/photocatalog /usr/local/bin/photocatalog
+```
+
+## Supported formats
+At this moment supported jpeg files with filled exif data or any other 
+files but with names matching pattern `yyymmdd_HHMMSS.ext`. Such 
+names format applied by android's camera software (I guess all cams 
+use this format, fix me if I'm wrong).
+
+There is not support for changing supported naming format without modifying  source code.
+
+## Usage
+### One-shot 
+#### Copy files (make a COW if fs supports it)
+```bash
+photocalog -mode copy -target ./photos/ ./sync/photos/*
+```
+
+#### Create hardlinks (only withing one disk partition)
+```bash
+photocalog -mode hardlink -target ./photos/ ./sync/photos/*
+```
+or 
+```bash
+photocalog -target ./photos/ ./sync/photos/*
+```
+
+### Monitor
+#### Copy files (make a COW if fs supports it)
+```bash
+photocalog -mode copy -target ./photos/ ./sync/photos/*
+```
+
+#### Create hardlinks (only withing one disk partition)
+```bash
+photocalog -mode hardlink -target ./photos/ -monitor ./sync/photos/
+```
+or 
+```bash
+photocalog -target ./photos/ -monitor ./sync/photos/
+```
+
+## Install and run monitor service
+
+### Systemd
+```bash
+sh ./init/install_service.sh systemd
+```
+This command will install unit file, create stub for its config and open
+editor to allow you edit configuration. Config file stored at 
+`$HOME/.config/photocatalog`.
+
+Then enable and start service
+```bash
+systemctl --user enable --now photocatalog
+```
+That's all. Now, if any file will be placed in directory, specified as `MONITOR`
+in config file, this file will be copied or hardlinked into the target dir
+under corresponding sub-dir. 
+
+## FAQ
+
+### Why this tool was created if there is awesome XXX tool?
+I had two good reasons:
+1. I wanted
+2. I can
+

cmd/photocatalog/photocatalog.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"os/signal"
+	"path"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/fsnotify/fsnotify"
+
+	"photocatalog/pkg/core"
+)
+
+func main() {
+	mode := flag.String("mode", "hardlink", "Manage mode: copy or hardlink")
+	target := flag.String("target", "./", "Root directory to organize files in")
+	monitor := flag.String("monitor", "", "Monitor specified folder for new files")
+
+	flag.Parse()
+	args := flag.Args()
+	log.Println("Using", *target, "as target and", *mode, "as mode")
+
+	var manageMode core.ManageMode
+	switch *mode {
+	case "copy":
+		manageMode = core.Copy
+	case "hardlink":
+		manageMode = core.Hardlink
+	default:
+		log.Fatalf("Invalid mode %s", *mode)
+	}
+
+	manager, err := core.NewManager(*target, manageMode)
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	if *monitor == "" {
+		processFiles(args, manager)
+	} else {
+		startMonitoring(*monitor, manager)
+	}
+}
+
+func processFiles(args []string, manager *core.Manager) {
+	var manageErr error
+	var gotErrors bool
+
+	if len(args) > 0 {
+		var err error
+		if len(args) == 1 && strings.HasSuffix(args[0], "/") {
+			args, err = filepath.Glob(args[0] + "*")
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+		log.Println("Processing", len(args), "files")
+		for _, f := range args {
+			manageErr = manager.Manage(f)
+			if manageErr != nil {
+				log.Println(manageErr)
+				gotErrors = true
+			}
+		}
+	} else {
+		log.Println("No input files")
+	}
+
+	if gotErrors {
+		log.Println("All files processed, got errors")
+	} else {
+		log.Println("All files processed without errors")
+	}
+}
+
+func startMonitoring(monitor string, manager *core.Manager) {
+	var manageErr error
+
+	if !path.IsAbs(monitor) {
+		cwd, err := os.Getwd()
+		if err != nil {
+			log.Fatalf("failed to get CWD: %s", err.Error())
+		}
+		monitor = path.Join(cwd, monitor)
+	}
+
+	log.Println("Monitoring", monitor)
+
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer func() {
+		closeErr := watcher.Close()
+		if closeErr != nil {
+			log.Println(closeErr)
+		}
+	}()
+
+	done := make(chan os.Signal)
+	signal.Notify(done, os.Interrupt, syscall.SIGTERM)
+
+	go func() {
+		for {
+			select {
+			case event, ok := <-watcher.Events:
+				if !ok {
+					return
+				}
+				if event.Op == fsnotify.Create {
+					if strings.HasSuffix(event.Name, "tmp") {
+						continue
+					}
+					manageErr = manager.Manage(event.Name)
+					if manageErr != nil {
+						log.Println(manageErr)
+					}
+				}
+			case err, ok := <-watcher.Errors:
+				log.Println("error:", err)
+				if !ok {
+					return
+				}
+			}
+		}
+	}()
+
+	err = watcher.Add(monitor)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	sig := <-done
+	log.Println("Monitoring stopped with", sig, "signal")
+}

go.mod

@@ -0,0 +1,10 @@
+module github.com/derfenix/photocatalog
+
+go 1.12
+
+require (
+	github.com/fsnotify/fsnotify v1.4.7
+	github.com/pkg/errors v0.8.1
+	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
+	golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa // indirect
+)

go.sum

@@ -0,0 +1,8 @@
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc=
+github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
+golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M=
+golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

init/install_service.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+INIT=${1:-systemd}
+
+CONFIG_PATH="${XDG_CONFIG_HOME:-$HOME/.config}"
+SETTINGS_PATH="${CONFIG_PATH}/photocatalog"
+
+SYSTEMD_UNIT_PATH="${CONFIG_PATH}/systemd/user/"
+
+if "${INIT}" == "systemd"
+then
+    cp ./init/systemd/photocatalog.service $SYSTEMD_UNIT_PATH/photocatalog.service
+    if test ! -f "${SETTINGS_PATH}"
+    then
+        echo "TARGET=<specify target dir>\nMONITOR=<specify dir to monitor>\nMODE=hardlink" > "${SETTINGS_PATH}"
+        ${EDITOR} "${SETTINGS_PATH}"
+        exit $?
+    else
+        exit 0
+    fi
+fi
+
+echo "Unknown init"
+exit 2

init/systemd/photocatalog.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Organize photo files, received from Syncthing or other syncing tools
+
+[Install]
+WantedBy=default.target
+
+[Service]
+Type=simple
+EnvironmentFile=/home/%u/.config/photocatalog
+ExecStart=photocatalog -mode $MODE -target $TARGET -monitor $MONITOR
+ExecStartPre=photocatalog -mode $MODE -target $TARGET ${MONITOR}

pkg/core/managemode.go

@@ -0,0 +1,10 @@
+package core
+
+//go:generate stringer -type ManageMode
+
+type ManageMode uint8
+
+const (
+	Copy ManageMode = iota
+	Hardlink
+)

pkg/core/managemode_string.go

@@ -0,0 +1,24 @@
+// Code generated by "stringer -type ManageMode"; DO NOT EDIT.
+
+package core
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[Copy-0]
+	_ = x[Hardlink-1]
+}
+
+const _ManageMode_name = "CopyHardlink"
+
+var _ManageMode_index = [...]uint8{0, 4, 12}
+
+func (i ManageMode) String() string {
+	if i >= ManageMode(len(_ManageMode_index)-1) {
+		return "ManageMode(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _ManageMode_name[_ManageMode_index[i]:_ManageMode_index[i+1]]
+}

pkg/core/manager.go

@@ -0,0 +1,134 @@
+package core
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"photocatalog/pkg/metadata"
+)
+
+type Manager struct {
+	TargetPath string
+	Mode       ManageMode
+
+	processor       func(fp, targetDir string) (string, error)
+	extractorsCache map[string]metadata.Extractor
+}
+
+func NewManager(target string, mode ManageMode) (*Manager, error) {
+	manager := Manager{
+		TargetPath: target,
+		Mode:       mode,
+		processor:  nil,
+	}
+	if err := manager.initProcessor(); err != nil {
+		return nil, err
+	}
+	manager.extractorsCache = map[string]metadata.Extractor{}
+	return &manager, nil
+}
+
+func (m *Manager) buildTarget(meta *metadata.Metadata) (string, error) {
+	dir, err := m.dirPathFromMeta(meta)
+	if err != nil {
+		return "", err
+	}
+	dirPath := path.Join(m.TargetPath, dir)
+	err = os.MkdirAll(dirPath, os.FileMode(0770))
+	if err != nil {
+		return "", err
+	}
+	return dirPath, nil
+}
+
+func (m *Manager) dirPathFromMeta(meta *metadata.Metadata) (string, error) {
+	t := meta.Time
+	year := t.Format("2006")
+	month := t.Format("01")
+	day := t.Format("02")
+	return path.Join(year, month, day), nil
+}
+
+func (m *Manager) getMetadataExtractor(fp string) metadata.Extractor {
+	switch strings.ToLower(path.Ext(fp)) {
+	case ".jpeg", ".jpg":
+		if _, ok := m.extractorsCache["jpeg"]; !ok {
+			m.extractorsCache["jpeg"] = metadata.NewJpegExtractor()
+		}
+		return m.extractorsCache["jpeg"]
+	default:
+		if _, ok := m.extractorsCache["default"]; !ok {
+			m.extractorsCache["default"] = metadata.NewDefaultExtractor()
+		}
+		return m.extractorsCache["default"]
+	}
+}
+
+func (m *Manager) initProcessor() error {
+	switch m.Mode {
+	case Copy:
+		m.processor = func(fp, targetDir string) (string, error) {
+			_, fn := path.Split(fp)
+			target := path.Join(targetDir, fn)
+			cmd := exec.Command("cp", "-f", "--reflink=auto", fp, target)
+			return target, cmd.Run()
+		}
+	case Hardlink:
+		m.processor = func(fp, targetDir string) (string, error) {
+			_, fn := path.Split(fp)
+			target := path.Join(targetDir, fn)
+			cmd := exec.Command("ln", "-f", fp, target)
+			return target, cmd.Run()
+		}
+	default:
+		return fmt.Errorf("failed to init processor: invalid Mode value")
+	}
+	return nil
+}
+
+func (m *Manager) Manage(fp string) error {
+	if m.processor == nil {
+		return fmt.Errorf("no processor initialized")
+	}
+	// Skip hidden files
+	if strings.HasPrefix(path.Base(fp), ".") {
+		return nil
+	}
+
+	log.Println("processing", fp)
+
+	extractor := m.getMetadataExtractor(fp)
+	if extractor == nil {
+		return fmt.Errorf("failed to get md extractor for %s", fp)
+	}
+
+	md, err := extractor.Extract(fp)
+	if err != nil {
+		return errors.WithMessagef(err, "failed to extract md from %s", fp)
+	}
+
+	targetDir, err := m.buildTarget(&md)
+	if err != nil {
+		return errors.WithMessagef(err, "failed to create dir for %s", fp)
+	}
+
+	target, err := m.processor(fp, targetDir)
+	if err != nil {
+		return errors.WithMessagef(err, "failed to process %s to %s", fp, targetDir)
+	}
+
+	if m.Mode == Hardlink {
+		log.Println(fp, "linked to", target)
+	} else if m.Mode == Copy {
+		log.Println(fp, "copied to", target)
+	}
+
+	log.Println("success")
+	return nil
+}

pkg/metadata/base.go

@@ -0,0 +1,15 @@
+package metadata
+
+import (
+	"time"
+)
+
+// Metadata contains meta data for the files have to be processed
+type Metadata struct {
+	Time time.Time
+}
+
+// Extractor interface for Metadata extractors
+type Extractor interface {
+	Extract(string) (Metadata, error)
+}

pkg/metadata/default.go

@@ -0,0 +1,45 @@
+package metadata
+
+import (
+	"path"
+	"strings"
+	"time"
+)
+
+const defaultTimeLayout = "20060102_150405"
+
+// DefaultExtractor extract metadata from all file types, not covered by special extractors
+//
+// Gets the meta data from the file's name
+type DefaultExtractor struct {
+	Layout string
+}
+
+// NewDefaultExtractor returns new DefaultExtractor's instance
+func NewDefaultExtractor() *DefaultExtractor {
+	return &DefaultExtractor{Layout: defaultTimeLayout}
+}
+
+// NewDefaultExtractorWithLayout returns DefaultExtractor with custom time layout
+func NewDefaultExtractorWithLayout(l string) *DefaultExtractor {
+	return &DefaultExtractor{Layout: l}
+}
+
+// Extract returns Metadata from specified filename using its name to parse Time
+func (d *DefaultExtractor) Extract(fp string) (Metadata, error) {
+	_, fName := path.Split(fp)
+
+	// Remove extension
+	fName = strings.Replace(fName, path.Ext(fName), "", 1)
+
+	// If there more than one photo in one second, cameras append ~N to the end of file name (before extension)
+	if strings.ContainsRune(fName, '~') {
+		fName = fName[:strings.IndexRune(fName, '~')]
+	}
+
+	t, err := time.ParseInLocation(d.Layout, fName, time.Local)
+	if err != nil {
+		return Metadata{}, err
+	}
+	return Metadata{Time: t}, nil
+}

pkg/metadata/default_test.go

@@ -0,0 +1,70 @@
+package metadata
+
+import (
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestDefaultExtractor_Extract(t *testing.T) {
+	local, err := time.LoadLocation("Europe/Moscow")
+	if err != nil {
+		t.Fatal(err)
+	}
+	time.Local = local
+
+	type args struct {
+		fp string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    Metadata
+		wantErr bool
+	}{
+		{
+			name: "Normal",
+			args: args{"20190321_120325.jpg"},
+			want: Metadata{
+				Time: func() time.Time {
+					tm, err := time.Parse(time.RFC3339, "2019-03-21T12:03:25+03:00")
+					if err != nil {
+						t.Fatal(err)
+					}
+					return tm
+				}(),
+			},
+			wantErr: false,
+		},
+		{
+			name: "Name with tilda",
+			args: args{"20190321_120325~2.jpg"},
+			want: Metadata{
+				Time: func() time.Time {
+					tm, err := time.Parse(time.RFC3339, "2019-03-21T12:03:25+03:00")
+					if err != nil {
+						t.Fatal(err)
+					}
+					return tm
+				}(),
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			d := NewDefaultExtractor()
+			if err != nil {
+				t.Fatal(err)
+			}
+			got, err := d.Extract(tt.args.fp)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Extract() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Extract() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/metadata/jpeg.go

@@ -0,0 +1,40 @@
+package metadata
+
+import (
+	"os"
+
+	"github.com/rwcarlsen/goexif/exif"
+)
+
+// JpegExtractor meta data extractor for the jpeg files
+type JpegExtractor struct {
+}
+
+// NewJpegExtractor returns new JpegExtractor
+func NewJpegExtractor() *JpegExtractor {
+	return &JpegExtractor{}
+}
+
+// Extract returns Metadata from specified jpeg file reading its exif data
+//
+// TODO: Fallback to default extractor on exif reading/parsing error
+func (j *JpegExtractor) Extract(fp string) (Metadata, error) {
+	f, err := os.Open(fp)
+	if err != nil {
+		return Metadata{}, err
+	}
+	x, err := exif.Decode(f)
+	if err != nil {
+		return Metadata{}, err
+	}
+
+	time, err := x.DateTime()
+	if err != nil {
+		return Metadata{}, err
+	}
+	meta := Metadata{
+		Time: time,
+	}
+
+	return meta, nil
+}