Browse Source

Parse image if exists

resolves #1
pull/22/head
Efertone 10 months ago
parent
commit
d827775fc3
Signed by: efertone GPG Key ID: 58E2D23885002DC5
27 changed files with 270 additions and 52 deletions
  1. +1
    -1
      cmd/kiki/command/addAccount.go
  2. +1
    -1
      cmd/kiki/command/addFeed.go
  3. +1
    -1
      cmd/kiki/command/addHashTag.go
  4. +1
    -1
      cmd/kiki/command/fetch.go
  5. +1
    -1
      cmd/kiki/command/listAccounts.go
  6. +1
    -1
      cmd/kiki/command/listFeeds.go
  7. +2
    -2
      cmd/kiki/command/preview.go
  8. +5
    -5
      cmd/kiki/command/publish.go
  9. +1
    -1
      cmd/kiki/command/version.go
  10. +1
    -1
      go.mod
  11. +6
    -3
      pkg/cleaner/remote_trackers_test.go
  12. +1
    -1
      pkg/database/configuration.go
  13. +1
    -3
      pkg/database/main.go
  14. +115
    -11
      pkg/misskey/client.go
  15. +6
    -6
      pkg/misskey/client_test.go
  16. +51
    -0
      pkg/misskey/file_create.go
  17. +8
    -0
      pkg/misskey/interface.go
  18. +1
    -0
      pkg/model/entry.go
  19. +3
    -0
      pkg/provider/atom.go
  20. +9
    -1
      pkg/provider/download.go
  21. +2
    -0
      pkg/provider/entry.go
  22. +16
    -0
      pkg/provider/image.go
  23. +4
    -1
      pkg/provider/rdf.go
  24. +4
    -1
      pkg/provider/rss.go
  25. +13
    -3
      pkg/publisher/misskey.go
  26. +11
    -2
      pkg/publisher/misskey_test.go
  27. +4
    -5
      pkg/publisher/publisher.go

+ 1
- 1
cmd/kiki/command/addAccount.go View File

@ -9,7 +9,7 @@ import (
// AddAccount command.
func AddAccount() *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: "add-account",
Short: "Add a new Account",
Run: func(cmd *cobra.Command, args []string) {


+ 1
- 1
cmd/kiki/command/addFeed.go View File

@ -10,7 +10,7 @@ import (
// AddFeed command.
func AddFeed() *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: "add-feed",
Short: "Add Feed to an Account",
Run: func(cmd *cobra.Command, args []string) {


+ 1
- 1
cmd/kiki/command/addHashTag.go View File

@ -10,7 +10,7 @@ import (
// AddHashTag command.
func AddHashTag() *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: "add-hashtag",
Short: "Add HashTag to a Feed",
Run: func(cmd *cobra.Command, args []string) {


+ 1
- 1
cmd/kiki/command/fetch.go View File

@ -12,7 +12,7 @@ import (
// Fetch command.
func Fetch() *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: "fetch",
Short: "Fetch all Feed contents",
Run: func(cmd *cobra.Command, args []string) {


+ 1
- 1
cmd/kiki/command/listAccounts.go View File

@ -10,7 +10,7 @@ import (
// ListAccounts command.
func ListAccounts() *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: "list-accounts",
Short: "List all accounts",
Run: func(cmd *cobra.Command, args []string) {


+ 1
- 1
cmd/kiki/command/listFeeds.go View File

@ -10,7 +10,7 @@ import (
// ListFeeds command.
func ListFeeds() *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: "list-feeds",
Short: "List all Feeds",
Run: func(cmd *cobra.Command, args []string) {


+ 2
- 2
cmd/kiki/command/preview.go View File

@ -10,7 +10,7 @@ import (
// Preview command.
func Preview() *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: "preview",
Short: "Preview desired content",
}
@ -22,7 +22,7 @@ func Preview() *cobra.Command {
// PreviewFetch : Preview -> Fetch command.
func PreviewFetch() *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: "fetch",
Short: "Preview fetch",
Run: func(cmd *cobra.Command, args []string) {


+ 5
- 5
cmd/kiki/command/publish.go View File

@ -13,13 +13,13 @@ import (
// Publish command.
func Publish() *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: "publish",
Short: "Publish new entries to Misskey",
Run: func(cmd *cobra.Command, args []string) {
for _, acc := range account.All() {
prov := publisher.NewTokenPublisherByName(acc.Publisher, acc.BaseURL, acc.APIToken)
if prov == nil {
pub := publisher.NewTokenPublisherByName(acc.Publisher, acc.BaseURL, acc.APIToken)
if pub == nil {
fmt.Fprintf(cmd.OutOrStderr(), "Unknown Publisher: %s\n", acc.Publisher)
continue
}
@ -48,12 +48,12 @@ func Publish() *cobra.Command {
)
}
err := prov.Publish(content)
err := pub.Publish(content, next.Image)
if err == nil {
entry.MarkAsPosted(next)
continue
}
fmt.Fprintf(cmd.OutOrStdout(), "[%s] %s\n", prov.Name(), err.Error())
fmt.Fprintf(cmd.OutOrStdout(), "[%s] %s\n", pub.Name(), err.Error())
}
}
},


+ 1
- 1
cmd/kiki/command/version.go View File

@ -9,7 +9,7 @@ import (
// Version command.
func Version() *cobra.Command {
var cmd = &cobra.Command{
cmd := &cobra.Command{
Use: "version",
Short: "Print the version number of Kiki",
Run: func(cmd *cobra.Command, args []string) {


+ 1
- 1
go.mod View File

@ -9,7 +9,7 @@ require (
github.com/spf13/viper v1.7.0
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
github.com/stretchr/testify v1.4.0 // indirect
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553
golang.org/x/sys v0.0.0-20200107162124-548cf772de50 // indirect
google.golang.org/appengine v1.6.5 // indirect
jaytaylor.com/html2text v0.0.0-20200412013138-3577fbdbcff7


+ 6
- 3
pkg/cleaner/remote_trackers_test.go View File

@ -21,13 +21,16 @@ func TestRemoveTrackers(t *testing.T) {
{"https://example.com/?utm_campaign=evil_campaign", "https://example.com/"},
{
"https://example.com/?query=asd&utm_campaign=evil_campaign",
"https://example.com/?query=asd"},
"https://example.com/?query=asd",
},
{
"https://example.com/?utm_campaign=evil_campaign&query=asd",
"https://example.com/?query=asd"},
"https://example.com/?query=asd",
},
{
"https://example.com/?query=asd&utm_campaign=evil_campaign&another=one",
"https://example.com/?query=asd&another=one"},
"https://example.com/?query=asd&another=one",
},
}
for _, testCase := range suit {


+ 1
- 1
pkg/database/configuration.go View File

@ -16,7 +16,7 @@ type ConnectionDetails struct {
// Configure bootstraps a Database Connection.
func Configure(details *ConnectionDetails) {
var build = make([]string, 0)
build := make([]string, 0)
if details.User != "" {
build = append(build, fmt.Sprintf("user=%s", details.User))


+ 1
- 3
pkg/database/main.go View File

@ -10,9 +10,7 @@ import (
_ "github.com/jinzhu/gorm/dialects/postgres"
)
var (
connectionString string //nolint:gochecknoglobals
)
var connectionString string //nolint:gochecknoglobals
// Database is a wrapper for gorm.DB.
type Database struct {


+ 115
- 11
pkg/misskey/client.go View File

@ -7,6 +7,8 @@ import (
"io/ioutil"
"net/http"
"time"
"golang.org/x/net/context"
)
// HTTPClient is a simple intreface for http.Client.
@ -17,7 +19,9 @@ type HTTPClient interface {
// ClientInterface is an interface to describe how a Client looks like.
// Mostly for Mocking. Or later if Misskey gets multiple API versions.
type ClientInterface interface {
CreateNote(content string) error
CreateNote(content string, file *FileCreateResponse) error
CreateFile(content []byte, name string) (*FileCreateResponse, error)
CreateFileFromURL(url string, name string) (*FileCreateResponse, error)
}
// Client is the main Misskey client struct.
@ -27,10 +31,8 @@ type Client struct {
HTTPClient HTTPClient
}
const (
// RequestTimout is the timeout of a request in seconds.
RequestTimout = 10
)
// RequestTimout is the timeout of a request in seconds.
const RequestTimout = 10
// NewClient creates a new Misskey Client.
func NewClient(baseURL, token string) *Client {
@ -47,7 +49,7 @@ func (c Client) url(path string) string {
return fmt.Sprintf("%s/api%s", c.BaseURL, path)
}
func (c Client) sendRequest(request *BaseRequest) error {
func (c Client) sendJSONRequest(request *BaseRequest) error {
request.SetAPIToken(c.Token)
requestBody, err := request.ToJSON()
@ -55,7 +57,12 @@ func (c Client) sendRequest(request *BaseRequest) error {
return err
}
req, err := http.NewRequest("POST", c.url(request.Path), bytes.NewBuffer(requestBody))
req, err := http.NewRequestWithContext(
context.Background(),
"POST",
c.url(request.Path),
bytes.NewBuffer(requestBody),
)
if err != nil {
return err
}
@ -64,14 +71,13 @@ func (c Client) sendRequest(request *BaseRequest) error {
req.Header.Set("User-Agent", "Kiki, News Delivery Service")
resp, err := c.HTTPClient.Do(req)
if err != nil {
return RequestError{Message: ResponseReadError, Origin: err}
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return RequestError{Message: ResponseReadBodyError, Origin: err}
}
@ -95,12 +101,110 @@ func (c Client) sendRequest(request *BaseRequest) error {
return UnknownError{Response: errorResponse}
}
func (c *Client) sendMultiPartRequest(request MultipartRequest, path string, response interface{}) error {
body, contentType := request.Multipart(c.Token)
req, err := http.NewRequestWithContext(context.Background(), "POST", c.url(path), body)
if err != nil {
return err
}
req.Header.Set("Content-Type", contentType)
req.Header.Set("User-Agent", "Kiki, News Delivery Service")
resp, err := c.HTTPClient.Do(req)
if err != nil {
return RequestError{Message: ResponseReadError, Origin: err}
}
defer resp.Body.Close()
rbody, err := ioutil.ReadAll(resp.Body)
if err != nil {
return RequestError{Message: ResponseReadBodyError, Origin: err}
}
if resp.StatusCode == http.StatusOK {
return json.Unmarshal(rbody, response)
}
var errorWrapper errorResponseWrapper
err = json.Unmarshal(rbody, &errorWrapper)
if err != nil {
return RequestError{Message: ErrorResponseParseError, Origin: err}
}
var errorResponse ErrorResponse
if err := json.Unmarshal(errorWrapper.Error, &errorResponse); err != nil {
return RequestError{Message: ErrorResponseParseError, Origin: err}
}
return UnknownError{Response: errorResponse}
}
// CreateNote sends a request to the Misskey server to create a note.
func (c *Client) CreateNote(content string) error {
func (c *Client) CreateNote(content string, file *FileCreateResponse) error {
fileIDs := []string{}
if file != nil {
fileIDs = append(fileIDs, file.ID)
}
request := &NoteCreateRequest{
Visibility: "public",
Text: content,
FileIDs: fileIDs,
}
return c.sendJSONRequest(&BaseRequest{Request: request, Path: "/notes/create"})
}
// CreateFile sends a request to the Misskey server to create a file.
func (c *Client) CreateFile(content []byte, name string) (*FileCreateResponse, error) {
request := &FileCreateRequest{
Name: name,
Content: content,
}
var response *FileCreateResponse
err := c.sendMultiPartRequest(request, "/drive/files/create", response)
return response, err
}
// CreateFileFromURL sends a request to the Misskey server to create a file from a URL.
func (c *Client) CreateFileFromURL(url, name string) (*FileCreateResponse, error) {
content := c.downloadFile(url)
request := &FileCreateRequest{
Name: name,
Content: content,
}
response := &FileCreateResponse{}
err := c.sendMultiPartRequest(request, "/drive/files/create", response)
return response, err
}
func (c *Client) downloadFile(url string) []byte {
req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil)
if err != nil {
return []byte{}
}
resp, err := c.HTTPClient.Do(req)
if err != nil {
return []byte{}
}
defer resp.Body.Close()
content, err := ioutil.ReadAll(resp.Body)
if err != nil {
return []byte{}
}
return c.sendRequest(&BaseRequest{Request: request, Path: "/notes/create"})
return content
}

+ 6
- 6
pkg/misskey/client_test.go View File

@ -38,7 +38,7 @@ func TestNewClient_NormalRequestContent(t *testing.T) {
client := misskey.NewClient("https://localhost", "thisistoken")
client.HTTPClient = mockClient
err := client.CreateNote("content")
err := client.CreateNote("content", nil)
if err != nil {
t.Errorf("Unexpected error = %s", err)
}
@ -53,7 +53,7 @@ func TestNewClient_RequestError(t *testing.T) {
client := misskey.NewClient("https://localhost", "thisistoken")
client.HTTPClient = mockClient
err := client.CreateNote("content")
err := client.CreateNote("content", nil)
if err == nil {
t.Error("Expected error, but never happened")
return
@ -80,7 +80,7 @@ func TestNewClient_ReadError(t *testing.T) {
client := misskey.NewClient("https://localhost", "thisistoken")
client.HTTPClient = mockClient
err := client.CreateNote("content")
err := client.CreateNote("content", nil)
if err == nil {
t.Error("Expected error, but never happened")
return
@ -105,7 +105,7 @@ func TestNewClient_ErrorResponseWrapper_Error(t *testing.T) {
client := misskey.NewClient("https://localhost", "thisistoken")
client.HTTPClient = mockClient
err := client.CreateNote("content")
err := client.CreateNote("content", nil)
if err == nil {
t.Error("Expected error, but never happened")
return
@ -130,7 +130,7 @@ func TestNewClient_ErrorResponseParse_Error(t *testing.T) {
client := misskey.NewClient("https://localhost", "thisistoken")
client.HTTPClient = mockClient
err := client.CreateNote("content")
err := client.CreateNote("content", nil)
if err == nil {
t.Error("Expected error, but never happened")
return
@ -162,7 +162,7 @@ func TestNewClient_ValidErrorResponse(t *testing.T) {
client := misskey.NewClient("https://localhost", "thisistoken")
client.HTTPClient = mockClient
err := client.CreateNote("content")
err := client.CreateNote("content", nil)
if err == nil {
t.Error("Expected error, but never happened")
return


+ 51
- 0
pkg/misskey/file_create.go View File

@ -0,0 +1,51 @@
package misskey
import (
"bytes"
"mime/multipart"
)
// FileCreateRequest represents a CreateNote request.
type FileCreateRequest struct {
*BaseRequest
FolderID string
Name string
Sensitive bool
Force bool
Content []byte
}
// Multipart retruns with the multipart body and FormDataContentType.
func (f *FileCreateRequest) Multipart(i string) (*bytes.Buffer, string) {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
// File
filePart, _ := writer.CreateFormFile("file", f.Name)
_, _ = filePart.Write(f.Content)
// API Token
tokenPart, _ := writer.CreateFormField("i")
_, _ = tokenPart.Write([]byte(i))
// Name
namePart, _ := writer.CreateFormField("name")
_, _ = namePart.Write([]byte(f.Name))
writer.Close()
return body, writer.FormDataContentType()
}
// FileCreateResponse is the response structure of a file creation request.
type FileCreateResponse struct {
ID string `json:"id"`
CreatedAt string `json:"createdAt"`
Name string `json:"name"`
Type string `json:"type"`
MD5 string `json:"md5"`
Size uint `json:"size"`
URL string `json:"url"`
FolderID string `json:"folderId"`
Sensitive bool `json:"isSensitive"`
}

+ 8
- 0
pkg/misskey/interface.go View File

@ -0,0 +1,8 @@
package misskey
import "bytes"
// MultipartRequest is an interface for multipart form requests.
type MultipartRequest interface {
Multipart(token string) (*bytes.Buffer, string)
}

+ 1
- 0
pkg/model/entry.go View File

@ -15,6 +15,7 @@ type Entry struct {
Title string
EntryID string
Content string
Image string
PostedAt *time.Time
}


+ 3
- 0
pkg/provider/atom.go View File

@ -34,6 +34,8 @@ func (a *Atom) Parse(body []byte) (entries []*Entry) {
}
for _, entry := range atom.Entries {
imageURL := findImageURL(entry.Content)
title, _ := html2text.FromString(entry.Title, html2text.Options{
OmitLinks: true,
PrettyTables: false,
@ -48,6 +50,7 @@ func (a *Atom) Parse(body []byte) (entries []*Entry) {
Link: cleaner.RemoveTrackers(entry.Link.HRef),
Title: title,
Content: text,
Image: imageURL,
})
}


+ 9
- 1
pkg/provider/download.go View File

@ -1,13 +1,21 @@
package provider
import (
"context"
"io/ioutil"
"net/http"
)
// Download fetches the content of an URI and returns with a parsed []mode.Entry.
func Download(url string) ([]byte, error) {
resp, err := http.Get(url) //nolint:gosec
client := &http.Client{}
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}


+ 2
- 0
pkg/provider/entry.go View File

@ -8,6 +8,7 @@ type Entry struct {
Title string
Link string
Content string
Image string
}
// ToModel returns with the model representation
@ -18,5 +19,6 @@ func (e Entry) ToModel() *model.Entry {
Title: e.Title,
Link: e.Link,
Content: e.Content,
Image: e.Image,
}
}

+ 16
- 0
pkg/provider/image.go View File

@ -0,0 +1,16 @@
package provider
import "regexp"
const matchLength = 2
func findImageURL(content []byte) string {
imgFilter := regexp.MustCompile("<img .*src=['\"]([^'\"]+)['\"]")
match := imgFilter.FindSubmatch(content)
if len(match) != matchLength {
return ""
}
return string(match[1])
}

+ 4
- 1
pkg/provider/rdf.go View File

@ -33,6 +33,8 @@ func (a *RDF) Parse(body []byte) (entries []*Entry) {
}
for _, entry := range rdf.Items {
imageURL := findImageURL(entry.Content.Encoded)
title, _ := html2text.FromString(entry.Title, html2text.Options{
OmitLinks: true,
PrettyTables: false,
@ -47,8 +49,9 @@ func (a *RDF) Parse(body []byte) (entries []*Entry) {
Link: cleaner.RemoveTrackers(entry.Link),
Title: title,
Content: text,
Image: imageURL,
})
}
return
return entries
}

+ 4
- 1
pkg/provider/rss.go View File

@ -33,6 +33,8 @@ func (a *RSS) Parse(body []byte) (entries []*Entry) {
}
for _, entry := range rss.Channel.Items {
imageURL := findImageURL(entry.Content)
title, _ := html2text.FromString(entry.Title, html2text.Options{
OmitLinks: true,
PrettyTables: false,
@ -47,8 +49,9 @@ func (a *RSS) Parse(body []byte) (entries []*Entry) {
Link: cleaner.RemoveTrackers(entry.Link),
Title: title,
Content: text,
Image: imageURL,
})
}
return
return entries
}

+ 13
- 3
pkg/publisher/misskey.go View File

@ -1,6 +1,10 @@
package publisher
import "gitea.code-infection.com/efertone/kiki/pkg/misskey"
import (
"path"
"gitea.code-infection.com/efertone/kiki/pkg/misskey"
)
// Misskey publisher.
type Misskey struct {
@ -24,6 +28,12 @@ func (m *Misskey) Name() string {
}
// Publish simply publishes a Note.
func (m *Misskey) Publish(content string) error {
return m.Client.CreateNote(content)
func (m *Misskey) Publish(content, image string) error {
var file *misskey.FileCreateResponse
if image != "" {
file, _ = m.Client.CreateFileFromURL(image, path.Base(image))
}
return m.Client.CreateNote(content, file)
}

+ 11
- 2
pkg/publisher/misskey_test.go View File

@ -3,16 +3,25 @@ package publisher_test
import (
"testing"
"gitea.code-infection.com/efertone/kiki/pkg/misskey"
"gitea.code-infection.com/efertone/kiki/pkg/publisher"
)
type MockMisskeyClient struct {
}
func (m MockMisskeyClient) CreateNote(content string) error {
func (m MockMisskeyClient) CreateNote(content string, file *misskey.FileCreateResponse) error {
return nil
}
func (m MockMisskeyClient) CreateFile(content []byte, name string) (*misskey.FileCreateResponse, error) {
return nil, nil
}
func (m MockMisskeyClient) CreateFileFromURL(url string, name string) (*misskey.FileCreateResponse, error) {
return nil, nil
}
func TestMisskeyPublisher(t *testing.T) {
pub := publisher.NewMisskey("https://localhost", "token")
@ -22,7 +31,7 @@ func TestMisskeyPublisher(t *testing.T) {
t.Errorf("publisher.Name(): expected = misskey; got = %s", pub.Name())
}
err := pub.Publish("test")
err := pub.Publish("test", "")
if err != nil {
t.Errorf("Unexpedted error: %s", err)
}


+ 4
- 5
pkg/publisher/publisher.go View File

@ -3,16 +3,15 @@ package publisher
// Interface is a common interface for publishers like Misskey.
type Interface interface {
Name() string
Publish(content string) error
Publish(content string, imageURL string) error
}
// NewTokenPublisherByName creates a new Publisher with token auth
// based on its name.
func NewTokenPublisherByName(name, baseURL, token string) Interface {
switch name {
case "misskey":
if name == misskeyName {
return NewMisskey(baseURL, token)
default:
return nil
}
return nil
}

Loading…
Cancel
Save