commit b69709e7f3cf8a19bf4918dbbd61a996ac224623
parent ef7329caea0bc3f91af7dc3a8f1e1a216876e6fb
Author: Vincent Demeester <vincent@sbr.pm>
Date: Wed, 12 Jun 2024 20:44:25 +0200
tools/go-org-readwise: fetch with pages…
… and add tags
Signed-off-by: Vincent Demeester <vincent@sbr.pm>
Diffstat:
3 files changed, 116 insertions(+), 33 deletions(-)
diff --git a/tools/go-org-readwise/internal/org/org.go b/tools/go-org-readwise/internal/org/org.go
@@ -1 +1,12 @@
package org
+
+/*
+For each results:
+- Define a filename (denote naming — gonna be weird but meh)
+- Detect if the file exists
+- If the file doesn't exist, create the file
+- If the file exist, append
+
+For the file format: org file with denote naming
+And use the update date to add new highlights
+*/
diff --git a/tools/go-org-readwise/internal/readwise/readwise.go b/tools/go-org-readwise/internal/readwise/readwise.go
@@ -1,30 +1,61 @@
package readwise
-// TODO: support pages
-
import (
"context"
"encoding/json"
+ "fmt"
"io"
"net/http"
+ "strings"
"time"
)
-func FetchFromAPI(ctx context.Context, apikey string, updateAfter *time.Time) (Export, error) {
- export := Export{}
- endpoint := "https://readwise.io/api/v2/export"
- if updateAfter != nil {
- endpoint = endpoint + "/?updateAfter=" + updateAfter.Format(time.RFC3339)
- }
+const (
+ exportEndpoint = "https://readwise.io/api/v2/export/?"
+ FormatUpdatedAfter = "2006-01-02T15:04:05"
+)
+func FetchFromAPI(ctx context.Context, apikey string, updateAfter *time.Time) ([]Result, error) {
+ results := []Result{}
httpClient := &http.Client{}
+ var e Export
+ var err error
+ var nextPageCursor *int = nil
+ for {
+ e, err = fetchExport(ctx, httpClient, apikey, updateAfter, nextPageCursor)
+ if err != nil {
+ return results, err
+ }
+ results = append(results, e.Results...)
+ nextPageCursor = e.NextPageCursor
+ if nextPageCursor == nil {
+ // No more pages to fetch, we get out
+ break
+ }
+ }
+
+ return results, nil
+}
+
+func fetchExport(ctx context.Context, client *http.Client, apikey string, updateAfter *time.Time, nextPageCursor *int) (Export, error) {
+ export := Export{}
+ endpoint := exportEndpoint
+ params := []string{}
+ if updateAfter != nil {
+ params = append(params, "updatedAfter="+updateAfter.Format(FormatUpdatedAfter))
+ }
+ if nextPageCursor != nil {
+ params = append(params, fmt.Sprintf("pageCursor=%d", *nextPageCursor))
+ }
+ endpoint = endpoint + strings.Join(params, "&&")
+ fmt.Println(endpoint)
req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
if err != nil {
return export, err
}
req.Header.Add("Authorization", "Token "+apikey)
- resp, err := httpClient.Do(req)
+ resp, err := client.Do(req)
if err != nil {
return export, err
}
diff --git a/tools/go-org-readwise/main.go b/tools/go-org-readwise/main.go
@@ -2,38 +2,79 @@ package main
import (
"context"
+ "flag"
"fmt"
+ "log"
"os"
+ "path/filepath"
"time"
"github.com/vdemeester/home/tools/go-org-readwise/internal/readwise"
)
func main() {
+ apiKeyFile := flag.String("apiKeyFile", "", "File to load the apiKey from. If empty, it will defer to the READWISE_KEY environment variable")
+ targetFolder := flag.String("targetFolder", "", "Folder to write highlights (in org file) into")
+ flag.Parse()
+
+ if *targetFolder == "" {
+ log.Fatal("-targetFolder is a required flag")
+ }
+
+ apiKeyData, err := os.ReadFile(*apiKeyFile)
+ if err != nil && !os.IsNotExist(err) {
+ log.Fatalf("Error reading apiKeyFile %s: %v", *apiKeyFile, err)
+ }
+ apikey := string(apiKeyData)
+ if apikey == "" {
+ apikey = os.Getenv("READWISE_KEY")
+ }
+
+ stateFile := filepath.Join(*targetFolder, ".readwise-sync.state")
+ updateAfter, err := getUpdateAfterFromFile(stateFile)
+ if err != nil {
+ log.Fatalf("Error reading readwise state file from %s: %v", stateFile, err)
+ }
+ fmt.Println(*targetFolder)
+ fmt.Println("updateAfter", updateAfter)
ctx := context.Background()
- highlights, merr := readwise.FetchFromAPI(ctx, os.Getenv("READWISE_KEY"), nil)
- if merr != nil {
- fmt.Fprintf(os.Stderr, "%v\n", merr)
- os.Exit(1)
- }
- fmt.Println("count", highlights.Count)
- fmt.Println("nextPageCursor", *highlights.NextPageCursor)
- fmt.Println("size", len(highlights.Results))
-
- updateAfter := time.Now().Add(-1000 * time.Hour)
- fmt.Println("updateAfter:", updateAfter)
- highlights, merr = readwise.FetchFromAPI(ctx, os.Getenv("READWISE_KEY"), &updateAfter)
- if merr != nil {
- fmt.Fprintf(os.Stderr, "%v\n", merr)
- os.Exit(1)
- }
- fmt.Println("count", highlights.Count)
- fmt.Println("nextPageCursor", *highlights.NextPageCursor)
- fmt.Println("size", len(highlights.Results))
- for _, h := range highlights.Results {
- fmt.Println("title", h.Title, len(h.Highlights), h.BookTags)
- // for _, hh := range h.Highlights {
- // fmt.Println(">>>", hh.ID, hh.Tags)
- // }
+ highlights, err := readwise.FetchFromAPI(ctx, apikey, updateAfter)
+ if err != nil {
+ log.Fatalf("Error while fetching highlights: %v", err)
+ }
+ // if err := os.WriteFile(stateFile, []byte(time.Now().Format(readwise.FormatUpdatedAfter)), 0o666); err != nil {
+ // log.Fatalf("Error writing readwise state file in %s: %v", stateFile, err)
+ // }
+ fmt.Println("size", len(highlights))
+
+ // updateAfter := time.Now().Add(-72 * time.Hour)
+ // fmt.Println("updateAfter:", updateAfter)
+ // mhighlights, merr := readwise.FetchFromAPI(ctx, os.Getenv("READWISE_KEY"), &updateAfter)
+ // if merr != nil {
+ // fmt.Fprintf(os.Stderr, "%v\n", merr)
+ // os.Exit(1)
+ // }
+ // fmt.Println("size", len(mhighlights))
+ // for _, h := range highlights {
+ // fmt.Println("title", h.Title, len(h.Highlights), h.BookTags)
+ // // for _, hh := range h.Highlights {
+ // // fmt.Println(">>>", hh.ID, hh.Tags)
+ // // }
+ // }
+}
+
+func getUpdateAfterFromFile(stateFile string) (*time.Time, error) {
+ data, err := os.ReadFile(stateFile)
+ if err != nil && !os.IsNotExist(err) {
+ return nil, err
+ }
+ // If the file doesn't exists, do not fail
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ t, err := time.Parse(readwise.FormatUpdatedAfter, string(data))
+ if err != nil {
+ return nil, err
}
+ return &t, nil
}
