From ccce5e26b332b6b31e1259b936159644bd1b9749 Mon Sep 17 00:00:00 2001 From: Steven Vandevelde Date: Sat, 15 Dec 2018 22:52:29 +0100 Subject: [PATCH] Add AmazonS3 service --- elm.json | 13 +- src/Library/Crypto/Hmac.elm | 2 +- src/Library/Dict/Ext.elm | 15 ++ src/Library/Maybe/Ext.elm | 13 ++ src/Library/Sources.elm | 14 +- src/Library/Sources/Pick.elm | 24 +++ src/Library/Sources/Processing.elm | 45 +++++ src/Library/Sources/Services.elm | 119 ++++++++++++ src/Library/Sources/Services/AmazonS3.elm | 168 +++++++++++++++++ .../Sources/Services/AmazonS3/Parser.elm | 62 +++++++ .../Sources/Services/AmazonS3/Presign.elm | 172 ++++++++++++++++++ src/Library/Sources/Services/Common.elm | 61 +++++++ src/Library/String/Ext.elm | 19 ++ 13 files changed, 721 insertions(+), 6 deletions(-) create mode 100644 src/Library/Dict/Ext.elm create mode 100644 src/Library/Maybe/Ext.elm create mode 100644 src/Library/Sources/Pick.elm create mode 100644 src/Library/Sources/Processing.elm create mode 100644 src/Library/Sources/Services.elm create mode 100644 src/Library/Sources/Services/AmazonS3.elm create mode 100644 src/Library/Sources/Services/AmazonS3/Parser.elm create mode 100644 src/Library/Sources/Services/AmazonS3/Presign.elm create mode 100644 src/Library/Sources/Services/Common.elm create mode 100644 src/Library/String/Ext.elm diff --git a/elm.json b/elm.json index 6a47d536..844db597 100644 --- a/elm.json +++ b/elm.json @@ -15,21 +15,26 @@ "elm/html": "1.0.0", "elm/http": "2.0.0", "elm/json": "1.1.2", + "elm/regex": "1.0.0", "elm/svg": "1.0.1", "elm/time": "1.0.0", "elm/url": "1.0.0", "elm-community/list-extra": "8.1.0", + "elm-community/maybe-extra": "5.0.0", "icidasset/elm-binary": "1.3.0", "icidasset/elm-sha": "1.0.0", "jorgengranseth/elm-string-format": "1.0.1", "justgage/tachyons-elm": "4.1.1", - "rtfeldman/elm-hex": "1.0.0" + "ryannhg/date-format": "2.3.0", + "ymtszw/elm-xml-decode": "3.0.0" }, "indirect": { "elm/bytes": "1.0.7", "elm/file": "1.0.1", - "elm/regex": "1.0.0", - "elm/virtual-dom": "1.0.2" + "elm/parser": "1.1.0", + "elm/virtual-dom": "1.0.2", + "jinjor/elm-xml-parser": "2.0.0", + "rtfeldman/elm-hex": "1.0.0" } }, "test-dependencies": { @@ -38,4 +43,4 @@ "elm/random": "1.0.0" } } -} +} \ No newline at end of file diff --git a/src/Library/Crypto/Hmac.elm b/src/Library/Crypto/Hmac.elm index 1e47b59e..f54c3d53 100644 --- a/src/Library/Crypto/Hmac.elm +++ b/src/Library/Crypto/Hmac.elm @@ -1,4 +1,4 @@ -module Crypto.Hmac exposing (encrypt128, encrypt64) +module Crypto.HMAC exposing (encrypt128, encrypt64) {-| Cryptography – HMAC -} diff --git a/src/Library/Dict/Ext.elm b/src/Library/Dict/Ext.elm new file mode 100644 index 00000000..d23147e5 --- /dev/null +++ b/src/Library/Dict/Ext.elm @@ -0,0 +1,15 @@ +module Dict.Ext exposing (fetch, fetchUnknown) + +import Dict exposing (Dict) + + +fetch : comparable -> v -> Dict comparable v -> v +fetch key default dict = + dict + |> Dict.get key + |> Maybe.withDefault default + + +fetchUnknown : comparable -> Dict comparable String -> String +fetchUnknown key dict = + fetch key "MISSING_VALUE" dict diff --git a/src/Library/Maybe/Ext.elm b/src/Library/Maybe/Ext.elm new file mode 100644 index 00000000..298a1322 --- /dev/null +++ b/src/Library/Maybe/Ext.elm @@ -0,0 +1,13 @@ +module Maybe.Ext exposing (preferFirst, preferSecond) + +import Maybe.Extra + + +preferFirst : Maybe a -> Maybe a -> Maybe a +preferFirst = + Maybe.Extra.or + + +preferSecond : Maybe a -> Maybe a -> Maybe a +preferSecond = + Maybe.Extra.orElse diff --git a/src/Library/Sources.elm b/src/Library/Sources.elm index 78f5f707..1c1574fc 100644 --- a/src/Library/Sources.elm +++ b/src/Library/Sources.elm @@ -1,4 +1,4 @@ -module Sources exposing (Form, FormStep(..), Page(..), Service(..), Source, SourceData, defaultService, emptySource, newForm) +module Sources exposing (Form, FormStep(..), Page(..), Property, Service(..), Source, SourceData, defaultService, emptySource, newForm) import Dict exposing (Dict) @@ -20,10 +20,22 @@ type alias Source = -- PIECES +type alias Property = + { prop : String + , labl : String + , plho : String + , pass : Bool + } + + type alias SourceData = Dict String String + +-- SERVICES + + type Service = AmazonS3 diff --git a/src/Library/Sources/Pick.elm b/src/Library/Sources/Pick.elm new file mode 100644 index 00000000..16a41a71 --- /dev/null +++ b/src/Library/Sources/Pick.elm @@ -0,0 +1,24 @@ +module Sources.Pick exposing (isMusicFile, selectMusicFiles) + +import Regex + + +isMusicFile : String -> Bool +isMusicFile = + Regex.contains musicFileRegex + + +selectMusicFiles : List String -> List String +selectMusicFiles = + List.filter isMusicFile + + + +-- PRIVATE + + +musicFileRegex : Regex.Regex +musicFileRegex = + "\\.(mp3|mp4|m4a|flac)$" + |> Regex.fromStringWith { caseInsensitive = True, multiline = False } + |> Maybe.withDefault Regex.never diff --git a/src/Library/Sources/Processing.elm b/src/Library/Sources/Processing.elm new file mode 100644 index 00000000..9ad4389a --- /dev/null +++ b/src/Library/Sources/Processing.elm @@ -0,0 +1,45 @@ +module Sources.Processing exposing (HttpMethod(..), Marker(..), PrepationAnswer, TreeAnswer, httpMethod) + +import Http +import Sources exposing (SourceData) + + + +-- MARKERS & RESPONSES + + +type Marker + = TheBeginning + | InProgress String + | TheEnd + + +type alias PrepationAnswer marker = + { sourceData : SourceData + , marker : marker + } + + +type alias TreeAnswer marker = + { filePaths : List String + , marker : marker + } + + + +-- HTTP + + +type HttpMethod + = Get + | Head + + +httpMethod : HttpMethod -> String +httpMethod method = + case method of + Get -> + "GET" + + Head -> + "HEAD" diff --git a/src/Library/Sources/Services.elm b/src/Library/Sources/Services.elm new file mode 100644 index 00000000..f9c41f7e --- /dev/null +++ b/src/Library/Sources/Services.elm @@ -0,0 +1,119 @@ +module Sources.Services exposing (initialData, keyToType, labels, makeTrackUrl, makeTree, parseErrorResponse, parsePreparationResponse, parseTreeResponse, postProcessTree, prepare, properties, typeToKey) + +{-| Service functions used in other modules. +-} + +import Http +import Sources exposing (..) +import Sources.Processing exposing (..) +import Sources.Services.AmazonS3 as AmazonS3 +import Time + + + +-- FUNCTIONS + + +initialData : Service -> SourceData +initialData service = + case service of + AmazonS3 -> + AmazonS3.initialData + + +makeTrackUrl : Service -> Time.Posix -> SourceData -> HttpMethod -> String -> String +makeTrackUrl service = + case service of + AmazonS3 -> + AmazonS3.makeTrackUrl + + +makeTree : + Service + -> SourceData + -> Marker + -> Time.Posix + -> (Result Http.Error String -> msg) + -> Cmd msg +makeTree service = + case service of + AmazonS3 -> + AmazonS3.makeTree + + +parseErrorResponse : Service -> String -> String +parseErrorResponse service = + case service of + AmazonS3 -> + AmazonS3.parseErrorResponse + + +parsePreparationResponse : Service -> String -> SourceData -> Marker -> PrepationAnswer Marker +parsePreparationResponse service = + case service of + AmazonS3 -> + AmazonS3.parsePreparationResponse + + +parseTreeResponse : Service -> String -> Marker -> TreeAnswer Marker +parseTreeResponse service = + case service of + AmazonS3 -> + AmazonS3.parseTreeResponse + + +postProcessTree : Service -> List String -> List String +postProcessTree service = + case service of + AmazonS3 -> + AmazonS3.postProcessTree + + +prepare : + Service + -> String + -> SourceData + -> Marker + -> (Result Http.Error String -> msg) + -> Maybe (Cmd msg) +prepare service = + case service of + AmazonS3 -> + AmazonS3.prepare + + +properties : Service -> List Property +properties service = + case service of + AmazonS3 -> + AmazonS3.properties + + + +-- KEYS & LABELS + + +keyToType : String -> Maybe Service +keyToType str = + case str of + "AmazonS3" -> + Just AmazonS3 + + _ -> + Nothing + + +typeToKey : Service -> String +typeToKey service = + case service of + AmazonS3 -> + "AmazonS3" + + +{-| Service labels. +Maps a service key to a label. +-} +labels : List ( String, String ) +labels = + [ ( typeToKey AmazonS3, "Amazon S3" ) + ] diff --git a/src/Library/Sources/Services/AmazonS3.elm b/src/Library/Sources/Services/AmazonS3.elm new file mode 100644 index 00000000..e34faee0 --- /dev/null +++ b/src/Library/Sources/Services/AmazonS3.elm @@ -0,0 +1,168 @@ +module Sources.Services.AmazonS3 exposing (defaults, initialData, makeTrackUrl, makeTree, parseErrorResponse, parsePreparationResponse, parseTreeResponse, postProcessTree, prepare, properties) + +{-| Amazon S3 Service. + +Resources: + + - + +-} + +import Dict +import Http +import Sources exposing (..) +import Sources.Pick +import Sources.Processing exposing (..) +import Sources.Services.AmazonS3.Parser as Parser +import Sources.Services.AmazonS3.Presign exposing (..) +import Sources.Services.Common exposing (cleanPath, nameProperty, noPrep) +import Time + + + +-- PROPERTIES +-- 📟 + + +defaults = + { directoryPath = "/" + , name = "Music from Amazon S3" + , region = "eu-west-1" + } + + +{-| The list of properties we need from the user. +-} +properties : List Property +properties = + [ { prop = "accessKey", labl = "Access key", plho = "Fv6EWfLfCcMo", pass = False } + , { prop = "secretKey", labl = "Secret key", plho = "qeNcqiMpgqC8", pass = True } + , { prop = "bucketName", labl = "Bucket name", plho = "music", pass = False } + , { prop = "region", labl = "Region", plho = defaults.region, pass = False } + , { prop = "directoryPath", labl = "Directory", plho = defaults.directoryPath, pass = False } + , { prop = "host", labl = "Host (optional)", plho = "http://127.0.0.1:9000", pass = False } + + -- + , nameProperty defaults.name + ] + + +{-| Initial data set. +-} +initialData : SourceData +initialData = + Dict.fromList + [ ( "accessKey", "" ) + , ( "bucketName", "" ) + , ( "directoryPath", defaults.directoryPath ) + , ( "host", "" ) + , ( "name", defaults.name ) + , ( "region", defaults.region ) + , ( "secretKey", "" ) + ] + + + +-- PREPARATION + + +prepare : String -> SourceData -> Marker -> (Result Http.Error String -> msg) -> Maybe (Cmd msg) +prepare _ _ _ _ = + Nothing + + + +-- TREE + + +{-| Create a directory tree. + +List all the tracks in the bucket. +Or a specific directory in the bucket. + +-} +makeTree : SourceData -> Marker -> Time.Posix -> (Result Http.Error String -> msg) -> Cmd msg +makeTree srcData marker currentTime resultMsg = + let + directoryPath = + srcData + |> Dict.get "directoryPath" + |> Maybe.withDefault defaults.directoryPath + |> cleanPath + + initialParams = + [ ( "list-type", "2" ) + , ( "max-keys", "1000" ) + ] + + prefix = + if String.length directoryPath > 0 then + [ ( "prefix", directoryPath ) ] + + else + [] + + continuation = + case marker of + InProgress s -> + [ ( "continuation-token", s ) ] + + _ -> + [] + + params = + initialParams ++ prefix ++ continuation + + url = + presignedUrl Get (60 * 5) params currentTime srcData "/" + in + Http.get + { url = url + , expect = Http.expectString resultMsg + } + + +{-| Re-export parser functions. +-} +parsePreparationResponse : String -> SourceData -> Marker -> PrepationAnswer Marker +parsePreparationResponse = + noPrep + + +parseTreeResponse : String -> Marker -> TreeAnswer Marker +parseTreeResponse = + Parser.parseTreeResponse + + +parseErrorResponse : String -> String +parseErrorResponse = + Parser.parseErrorResponse + + + +-- Post + + +{-| Post process the tree results. + +Make sure we only use music files that we can use. + +-} +postProcessTree : List String -> List String +postProcessTree = + Sources.Pick.selectMusicFiles + + + +-- Track URL + + +{-| Create a public url for a file. + +We need this to play the track. +Creates a presigned url that's valid for 48 hours + +-} +makeTrackUrl : Time.Posix -> SourceData -> HttpMethod -> String -> String +makeTrackUrl currentTime srcData method pathToFile = + presignedUrl method 172800 [] currentTime srcData pathToFile diff --git a/src/Library/Sources/Services/AmazonS3/Parser.elm b/src/Library/Sources/Services/AmazonS3/Parser.elm new file mode 100644 index 00000000..f540513f --- /dev/null +++ b/src/Library/Sources/Services/AmazonS3/Parser.elm @@ -0,0 +1,62 @@ +module Sources.Services.AmazonS3.Parser exposing (parseErrorResponse, parseTreeResponse) + +import Conditional exposing (..) +import Sources.Processing exposing (Marker(..), TreeAnswer) +import Xml.Decode exposing (..) + + + +-- TREE + + +parseTreeResponse : String -> Marker -> TreeAnswer Marker +parseTreeResponse response _ = + response + |> decodeString + (map2 + (\f m -> { filePaths = f, marker = m }) + filePathsDecoder + markerDecoder + ) + |> Result.withDefault { filePaths = [], marker = TheEnd } + + +filePathsDecoder : Decoder (List String) +filePathsDecoder = + path [ "Contents" ] (list <| path [ "Key" ] (single string)) + + +markerDecoder : Decoder Marker +markerDecoder = + map2 + (\a b -> + Maybe.withDefault + TheEnd + (Maybe.map2 + (\isTruncated token -> + ifThenElse (isTruncated == "true") (InProgress token) TheEnd + ) + a + b + ) + ) + (maybe <| path [ "IsTruncated" ] (single string)) + (maybe <| path [ "NextContinuationToken" ] (single string)) + + + +-- ERROR + + +parseErrorResponse : String -> String +parseErrorResponse response = + response + |> decodeString errorMessagesDecoder + |> Result.toMaybe + |> Maybe.andThen List.head + |> Maybe.withDefault "Invalid request" + + +errorMessagesDecoder : Decoder (List String) +errorMessagesDecoder = + path [ "Error" ] (list <| path [ "Message" ] (single string)) diff --git a/src/Library/Sources/Services/AmazonS3/Presign.elm b/src/Library/Sources/Services/AmazonS3/Presign.elm new file mode 100644 index 00000000..5c30caac --- /dev/null +++ b/src/Library/Sources/Services/AmazonS3/Presign.elm @@ -0,0 +1,172 @@ +module Sources.Services.AmazonS3.Presign exposing (presignedUrl) + +import Binary +import Crypto.Binary as Binary +import Crypto.HMAC as HMAC +import DateFormat as Date +import Dict +import Dict.Ext as Dict +import Maybe.Extra as Maybe +import SHA +import Sources exposing (SourceData) +import Sources.Processing exposing (HttpMethod, httpMethod) +import String.Ext as String +import Time +import Url +import Url.Builder as Url + + +presignedUrl : + HttpMethod + -> Int + -> List ( String, String ) + -> Time.Posix + -> SourceData + -> String + -> String +presignedUrl method lifeExpectancyInSeconds extraParams currentTime srcData pathToFile = + let + aws = + srcData + + region = + Dict.fetchUnknown "region" aws + + bucketName = + Dict.fetchUnknown "bucketName" aws + + customHost = + Dict.get "host" aws + + host = + case customHost of + Just h -> + h + |> String.chopStart "http://" + |> String.chopStart "https://" + |> String.chopEnd "/" + + Nothing -> + bucketName ++ ".s3.amazonaws.com" + + protocol = + if String.contains "http://" (Maybe.withDefault "" customHost) then + "http://" + + else + "https://" + + -- {var} Paths + filePathPrefix = + if Maybe.isJust customHost then + "/" ++ bucketName + + else + "" + + filePath = + pathToFile + |> String.chopStart "/" + |> String.split "/" + |> List.map Url.percentEncode + |> String.join "/" + |> String.append ("/" ++ filePathPrefix) + + -- {var} Time + -- timestamp -> 20130721T201207Z + -- date -> 20130721 + timestamp = + Date.format + [ Date.yearNumber + , Date.monthFixed + , Date.dayOfMonthFixed + , Date.text "T" + , Date.hourMilitaryFixed + , Date.minuteFixed + , Date.secondFixed + , Date.text "Z" + ] + Time.utc + currentTime + + date = + Date.format + [ Date.yearNumber + , Date.monthFixed + , Date.dayOfMonthFixed + ] + Time.utc + currentTime + + -- Request + credential = + [ Dict.fetchUnknown "accessKey" aws + , date + , region + , "s3" + , "aws4_request" + ] + |> String.join "/" + + queryString = + [ ( "X-Amz-Algorithm", "AWS4-HMAC-SHA256" ) + , ( "X-Amz-Credential", credential ) + , ( "X-Amz-Date", timestamp ) + , ( "X-Amz-Expires", String.fromInt lifeExpectancyInSeconds ) + , ( "X-Amz-SignedHeaders", "host" ) + ] + |> List.append extraParams + |> List.sortBy Tuple.first + |> List.map (\( a, b ) -> Url.string a b) + |> Url.toQuery + + request = + String.join + "\n" + [ httpMethod method + , filePath + , queryString + , "host:" ++ host + , "" + , "host" + , "UNSIGNED-PAYLOAD" + ] + + -- String to sign + stringToSign = + String.join + "\n" + [ "AWS4-HMAC-SHA256" + , timestamp + , String.join "/" [ date, region, "s3", "aws4_request" ] + , Binary.toHex (SHA.sha256 request) + ] + + -- Signature + signature = + ("AWS4" ++ Dict.fetchUnknown "secretKey" aws) + |> hmacSha256 date + |> hmacSha256 region + |> hmacSha256 "s3" + |> hmacSha256 "aws4_request" + |> hmacSha256 stringToSign + in + String.concat + [ protocol + , host + , filePath + , queryString + , "&X-Amz-Signature=" + , signature + ] + + + +-- ⚗️ + + +hmacSha256 : String -> String -> String +hmacSha256 message key = + key + |> HMAC.encrypt64 SHA.sha256 message + |> Binary.toString diff --git a/src/Library/Sources/Services/Common.elm b/src/Library/Sources/Services/Common.elm new file mode 100644 index 00000000..7aec0ca8 --- /dev/null +++ b/src/Library/Sources/Services/Common.elm @@ -0,0 +1,61 @@ +module Sources.Services.Common exposing (cleanPath, nameProperty, noPrep) + +import Sources exposing (..) +import Sources.Processing exposing (..) +import String.Ext as String + + + +-- FORMS + + +nameProperty : String -> Property +nameProperty placeholder = + { prop = "name" + , labl = "Name" + , plho = placeholder + , pass = False + } + + + +-- PATHS + + +{-| Clean a path. + + >>> cleanPath " " + "" + + >>> cleanPath "/example" + "example/" + + >>> cleanPath "example" + "example/" + + >>> cleanPath "example/" + "example/" + +-} +cleanPath : String -> String +cleanPath dirtyPath = + dirtyPath + |> String.trim + |> String.chopStart "/" + |> String.chopEnd "/" + |> (\p -> + if String.isEmpty p then + p + + else + p ++ "/" + ) + + + +-- PARSING + + +noPrep : String -> SourceData -> Marker -> PrepationAnswer Marker +noPrep _ srcData _ = + { sourceData = srcData, marker = TheEnd } diff --git a/src/Library/String/Ext.elm b/src/Library/String/Ext.elm new file mode 100644 index 00000000..fb20e39f --- /dev/null +++ b/src/Library/String/Ext.elm @@ -0,0 +1,19 @@ +module String.Ext exposing (chopEnd, chopStart) + + +chopEnd : String -> String -> String +chopEnd needle str = + if String.endsWith needle str then + String.dropRight (String.length str) str + + else + str + + +chopStart : String -> String -> String +chopStart needle str = + if String.startsWith needle str then + String.dropLeft (String.length str) str + + else + str -- 2.51.2