diff --git a/package-lock.json b/package-lock.json index e1cf11fa..84519478 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2515,10 +2515,11 @@ "optional": true }, "jsmediatags": { - "version": "http://registry.npmjs.org/jsmediatags/-/jsmediatags-3.5.0.tgz", - "integrity": "sha1-6OQypzlEhHEa1z7gWVheLoZAI+8=", + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/jsmediatags/-/jsmediatags-3.6.1.tgz", + "integrity": "sha1-AGZKOgP5HBj9Wc3TGugDUM1BPKE=", "requires": { - "xhr2": "http://registry.npmjs.org/xhr2/-/xhr2-0.1.4.tgz" + "xhr2": "0.1.4" } }, "json-schema": { @@ -4137,7 +4138,8 @@ "dev": true }, "xhr2": { - "version": "http://registry.npmjs.org/xhr2/-/xhr2-0.1.4.tgz", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/xhr2/-/xhr2-0.1.4.tgz", "integrity": "sha1-f4dliEdxbbUCYyOBL4GMras4el8=" }, "xtend": { diff --git a/package.json b/package.json index 2182ed33..c0fdd0f8 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "blockstack": "^0.9.0", "blockstack-storage": "blockstack/blockstack-storage-js#28b536057e6410a0195ae4e0b4ac36d358dec7ff", "elasticlunr": "^0.9.5", - "jsmediatags": "^3.5.0", + "jsmediatags": "^3.6.1", "lodash": "^4.17.4", "surge": "^0.19.0" } diff --git a/src/App/Sources/Encoding.elm b/src/App/Sources/Encoding.elm index 8382ff5b..28f12bca 100644 --- a/src/App/Sources/Encoding.elm +++ b/src/App/Sources/Encoding.elm @@ -9,11 +9,6 @@ import Json.Encode as Encode import Sources.Types exposing (..) --- Services - -import Sources.Services.AmazonS3 as AmazonS3 - - -- Encode @@ -63,5 +58,8 @@ serviceStringToType str = "AmazonS3" -> AmazonS3 + "Ipfs" -> + Ipfs + _ -> AmazonS3 diff --git a/src/App/Sources/Processing.elm b/src/App/Sources/Processing.elm index 1040156d..2cd47dae 100644 --- a/src/App/Sources/Processing.elm +++ b/src/App/Sources/Processing.elm @@ -23,12 +23,12 @@ module Sources.Processing -> This also happens in multiple steps, so that we can flush every x tracks while processing. A command is issued for each step of this process. + -} import Date exposing (Date) -import List.Extra as List +import List.Extra as List exposing (remove) import Maybe.Extra as Maybe -import Regex import Sources.Ports as Ports import Sources.Types exposing (..) import Tracks.Types exposing (TagUrls, Track, makeTrack) @@ -38,16 +38,18 @@ import Utils exposing (do) -- Services import Sources.Services.AmazonS3 as AmazonS3 +import Sources.Services.Ipfs as Ipfs -- Settings {-| How much tags do we want to process - before we send them back to Elm. +before we send them back to Elm. eg. After we got the tags for 50 tracks, we store these and continue with the rest. + -} tagsBatchSize : Int tagsBatchSize = @@ -85,33 +87,22 @@ takeTreeStep context response associatedTracks currentDate = TheEnd -> let - filteredContext = - selectMusicFiles newContext + filteredFiles = + case newContext.source.service of + AmazonS3 -> + AmazonS3.postProcessTree newContext.filePaths + + Ipfs -> + Ipfs.postProcessTree newContext.filePaths - remove = - \list path -> - list - |> List.findIndex (\x -> x == path) - |> Maybe.map (\idx -> List.removeAt idx list) - |> Maybe.withDefault list + postContext = + { newContext | filePaths = filteredFiles } ( pathsLeft, pathsToRemove, _ ) = - List.foldr - (\track ( left, toRemove, srcOfTruth ) -> - let - path = - track.path - in - if List.member path srcOfTruth then - ( path :: left, toRemove, remove srcOfTruth path ) - else - ( left, path :: toRemove, srcOfTruth ) - ) - ( [], [], filteredContext.filePaths ) - associatedTracks + separateTree postContext associatedTracks in Cmd.batch - [ filteredContext + [ postContext |> selectNonExisting pathsLeft |> processingContextToTagsContext |> ProcessTagsStep @@ -153,8 +144,11 @@ handleTreeResponse context response = AmazonS3 -> AmazonS3.parseTreeResponse + Ipfs -> + Ipfs.parseTreeResponse + parsedResponse = - parsingFunc response + parsingFunc response context.treeMarker in { context | filePaths = context.filePaths ++ parsedResponse.filePaths @@ -169,10 +163,30 @@ makeTree context = case context.source.service of AmazonS3 -> AmazonS3.makeTree + + Ipfs -> + Ipfs.makeTree in fn context.source.data context.treeMarker (ProcessTreeStep context) +separateTree : ProcessingContext -> List Track -> ( List String, List String, List String ) +separateTree context tracks = + List.foldr + (\track ( left, toRemove, srcOfTruth ) -> + let + path = + track.path + in + if List.member path srcOfTruth then + ( path :: left, toRemove, remove path srcOfTruth ) + else + ( left, path :: toRemove, srcOfTruth ) + ) + ( [], [], context.filePaths ) + tracks + + -- Tags @@ -185,6 +199,9 @@ makeTrackUrls currentDate source filePaths = AmazonS3 -> AmazonS3.makeTrackUrl + Ipfs -> + Ipfs.makeTrackUrl + mapFn = \path -> { getUrl = maker currentDate source.data Get path @@ -194,17 +211,6 @@ makeTrackUrls currentDate source filePaths = List.map mapFn filePaths -selectMusicFiles : ProcessingContext -> ProcessingContext -selectMusicFiles context = - let - regex = - Regex.regex "\\.(mp3|mp4|m4a)$" - in - { context - | filePaths = List.filter (Regex.contains regex) context.filePaths - } - - selectNonExisting : List String -> ProcessingContext -> ProcessingContext selectNonExisting existingPaths context = let @@ -226,6 +232,9 @@ decodeError source = AmazonS3 -> AmazonS3.parseErrorResponse + Ipfs -> + Ipfs.parseErrorResponse + findTagsContextSource : ProcessingContextForTags -> List Source -> Maybe Source findTagsContextSource tagsContext = @@ -238,6 +247,9 @@ makeTrackUrl currentDate source filePath = AmazonS3 -> AmazonS3.makeTrackUrl currentDate source.data Get filePath + Ipfs -> + Ipfs.makeTrackUrl currentDate source.data Get filePath + tracksFromTagsContext : ProcessingContextForTags -> List Track tracksFromTagsContext context = diff --git a/src/App/Sources/Services/AmazonS3.elm b/src/App/Sources/Services/AmazonS3.elm index 09ef7d7d..62528375 100644 --- a/src/App/Sources/Services/AmazonS3.elm +++ b/src/App/Sources/Services/AmazonS3.elm @@ -3,7 +3,8 @@ module Sources.Services.AmazonS3 exposing (..) {-| Amazon S3 Service. Resources: -- http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html + + - -} @@ -14,6 +15,7 @@ import Regex import Sources.Services.AmazonS3.Parser as Parser import Sources.Services.AmazonS3.Presign exposing (..) import Sources.Types exposing (..) +import Sources.Utils import Time @@ -29,8 +31,10 @@ defaults = {-| The list of properties we need from the user. - `(property, label, placeholder, isPassword)` - Will be used for the forms. + +Tuple: (property, label, placeholder, isPassword) +Will be used for the forms. + -} properties : List ( String, String, String, Bool ) properties = @@ -62,8 +66,10 @@ initialData = {-| Create a public url for a file. - We need this to play the track. - (!) Creates a presigned url that's valid for 24 hours + +We need this to play the track. +(!) Creates a presigned url that's valid for 24 hours + -} makeTrackUrl : Date -> SourceData -> HttpMethod -> String -> String makeTrackUrl currentDate srcData method pathToFile = @@ -75,8 +81,10 @@ makeTrackUrl currentDate srcData method pathToFile = {-| Create a directory tree. - List all the tracks in the bucket. - Or a specific directory in the bucket. + +List all the tracks in the bucket. +Or a specific directory in the bucket. + -} makeTree : SourceData -> Marker -> (TreeStepResult -> msg) -> Date -> Cmd msg makeTree srcData marker msg currentDate = @@ -126,7 +134,7 @@ makeTree srcData marker msg currentDate = {-| Re-export parser functions. -} -parseTreeResponse : String -> ParsedResponse Marker +parseTreeResponse : String -> Marker -> ParsedResponse Marker parseTreeResponse = Parser.parseTreeResponse @@ -134,3 +142,17 @@ parseTreeResponse = parseErrorResponse : String -> String parseErrorResponse = Parser.parseErrorResponse + + + +-- Post + + +{-| Post process the tree results. + +!!! Make sure we only use music files that we can use. + +-} +postProcessTree : List String -> List String +postProcessTree = + Sources.Utils.selectMusicFiles diff --git a/src/App/Sources/Services/AmazonS3/Parser.elm b/src/App/Sources/Services/AmazonS3/Parser.elm index b06289bb..76a65194 100644 --- a/src/App/Sources/Services/AmazonS3/Parser.elm +++ b/src/App/Sources/Services/AmazonS3/Parser.elm @@ -12,8 +12,8 @@ import Xml.Query exposing (..) -- Tree -parseTreeResponse : String -> ParsedResponse Marker -parseTreeResponse response = +parseTreeResponse : String -> Marker -> ParsedResponse Marker +parseTreeResponse response _ = let decodedXml = response diff --git a/src/App/Sources/Services/Ipfs.elm b/src/App/Sources/Services/Ipfs.elm new file mode 100644 index 00000000..ddb95193 --- /dev/null +++ b/src/App/Sources/Services/Ipfs.elm @@ -0,0 +1,151 @@ +module Sources.Services.Ipfs exposing (..) + +{-| IPFS Service. + +Resources: + + - + +-} + +import Date exposing (Date) +import Dict +import Http +import Sources.Services.Ipfs.Marker as Marker +import Sources.Services.Ipfs.Parser as Parser +import Sources.Types exposing (..) + + +-- Properties +-- 📟 + + +defaults = + { gateway = "http://localhost:8080" + , name = "IPFS source" + } + + +{-| The list of properties we need from the user. + +Tuple: (property, label, placeholder, isPassword) +Will be used for the forms. + +-} +properties : List ( String, String, String, Bool ) +properties = + [ ( "directoryHash", "Directory object hash", "QmVLDAhCY3X9P2u", False ) + , ( "gateway", "Read-only gateway", defaults.gateway, False ) + , ( "name", "Label", defaults.name, False ) + ] + + +{-| Initial data set. +-} +initialData : SourceData +initialData = + Dict.fromList + [ ( "directoryHash", "" ) + , ( "name", defaults.name ) + , ( "gateway", defaults.gateway ) + ] + + + +-- Track URL + + +{-| Create a public url for a file. + +We need this to play the track. + +-} +makeTrackUrl : Date -> SourceData -> HttpMethod -> String -> String +makeTrackUrl currentDate srcData _ hash = + let + gateway = + srcData + |> Dict.get "gateway" + |> Maybe.withDefault defaults.gateway + |> String.foldr + (\char acc -> + if String.isEmpty acc && char == '/' then + acc + else + String.cons char acc + ) + "" + in + gateway ++ "/ipfs/" ++ hash + + + +-- Tree + + +{-| Create a directory tree. + +List all the tracks in the bucket. +Or a specific directory in the bucket. + +-} +makeTree : SourceData -> Marker -> (TreeStepResult -> msg) -> Date -> Cmd msg +makeTree srcData marker msg currentDate = + let + gateway = + srcData + |> Dict.get "gateway" + |> Maybe.withDefault defaults.gateway + |> String.foldr + (\char acc -> + if String.isEmpty acc && char == '/' then + acc + else + String.cons char acc + ) + "" + + hash = + case marker of + InProgress progress -> + marker + |> Marker.takeOne + |> Maybe.withDefault "MISSING_HASH" + + _ -> + srcData + |> Dict.get "directoryHash" + |> Maybe.withDefault "MISSING_HASH" + + url = + gateway ++ "/api/v0/ls?arg=" ++ hash ++ "&encoding=json" + in + url + |> Http.getString + |> Http.send msg + + +{-| Re-export parser functions. +-} +parseTreeResponse : String -> Marker -> ParsedResponse Marker +parseTreeResponse = + Parser.parseTreeResponse + + +parseErrorResponse : String -> String +parseErrorResponse = + identity + + + +-- Post + + +{-| Post process the tree results. + +!!! Make sure we only use music files that we can use. + +-} +postProcessTree : List String -> List String +postProcessTree = + identity diff --git a/src/App/Sources/Services/Ipfs/Marker.elm b/src/App/Sources/Services/Ipfs/Marker.elm new file mode 100644 index 00000000..3f66064e --- /dev/null +++ b/src/App/Sources/Services/Ipfs/Marker.elm @@ -0,0 +1,93 @@ +module Sources.Services.Ipfs.Marker exposing (..) + +{-| Marker stuff for IPFS. + +The IPFS API currently doesn't have a way to return a tree. +So we have build one ourselves. + +How it works: + +1. We list the objects in a given directory +2. We make a list of the sub directories +3. The marker becomes either: + - InProgress `hashOfSubDirA/hashOfSubDirB/hashOfSubDirC` + - TheEnd +4. If the marker was of the type `InProgress`, + the next request will make a list of the objects in `hashOfSubDirA`. + And so on ... + +-} + +import Sources.Types exposing (Marker(..)) + + +separator : String +separator = + "/" + + + +-- In + + +concat : List String -> Marker -> Marker +concat list marker = + let + result = + case marker of + InProgress m -> + [ list, String.split separator m ] + |> List.concat + |> String.join separator + + _ -> + String.join separator list + in + case result of + "" -> + TheEnd + + r -> + InProgress r + + + +-- Out + + +{-| Take the first item and return it. +-} +takeOne : Marker -> Maybe String +takeOne marker = + case marker of + InProgress m -> + m + |> String.split separator + |> List.head + + _ -> + Nothing + + +{-| Remove the first item if there is one. +-} +removeOne : Marker -> Marker +removeOne marker = + case marker of + InProgress m -> + let + tmp = + m + |> String.split separator + |> List.drop 1 + |> String.join separator + in + case tmp of + "" -> + TheEnd + + x -> + InProgress x + + _ -> + TheEnd diff --git a/src/App/Sources/Services/Ipfs/Parser.elm b/src/App/Sources/Services/Ipfs/Parser.elm new file mode 100644 index 00000000..f4c7876b --- /dev/null +++ b/src/App/Sources/Services/Ipfs/Parser.elm @@ -0,0 +1,64 @@ +module Sources.Services.Ipfs.Parser exposing (..) + +import Json.Decode exposing (..) +import Sources.Services.Ipfs.Marker as Marker +import Sources.Types exposing (Marker(..), ParsedResponse) +import Sources.Utils exposing (isMusicFile) + + +-- Tree + + +parseTreeResponse : String -> Marker -> ParsedResponse Marker +parseTreeResponse response previousMarker = + let + links = + case decodeString treeDecoder response of + Ok l -> + l + + Err _ -> + [] + + dirs = + links + |> List.filter (.typ >> (==) 1) + |> List.map .hash + + files = + links + |> List.filter (.typ >> (==) 2) + |> List.filter (.name >> isMusicFile) + |> List.map .hash + in + { filePaths = + files + , marker = + previousMarker + |> Marker.removeOne + |> Marker.concat dirs + } + + +treeDecoder : Decoder (List Link) +treeDecoder = + field "Objects" <| index 0 <| field "Links" <| list linkDecoder + + + +-- Links + + +type alias Link = + { hash : String + , name : String + , typ : Int + } + + +linkDecoder : Decoder Link +linkDecoder = + map3 Link + (field "Hash" string) + (field "Name" string) + (field "Type" int) diff --git a/src/App/Sources/State.elm b/src/App/Sources/State.elm index e1147a9a..4067af2e 100644 --- a/src/App/Sources/State.elm +++ b/src/App/Sources/State.elm @@ -19,6 +19,7 @@ import Utils exposing (do) -- Services import Sources.Services.AmazonS3 as AmazonS3 +import Sources.Services.Ipfs as Ipfs -- 💧 @@ -28,7 +29,7 @@ initialModel : TopLevel.ProgramFlags -> Model initialModel flags = { collection = decodeSources flags , isProcessing = Nothing - , newSource = makeSource AmazonS3 AmazonS3.initialData + , newSource = makeSource Ipfs Ipfs.initialData , processingErrors = [] , timestamp = Date.fromTime 0 } @@ -208,7 +209,22 @@ update msg model = SetNewSourceProperty source key value -> let newSource = - { source | data = Dict.insert key value source.data } + { source | data = Dict.insert key (String.trim value) source.data } + in + (!) { model | newSource = newSource } [] + + SetNewSourceType typeString -> + let + newSource = + case typeString of + "AmazonS3" -> + makeSource AmazonS3 AmazonS3.initialData + + "Ipfs" -> + makeSource Ipfs Ipfs.initialData + + _ -> + makeSource AmazonS3 AmazonS3.initialData in (!) { model | newSource = newSource } [] diff --git a/src/App/Sources/Types.elm b/src/App/Sources/Types.elm index b01fcc35..60d56108 100644 --- a/src/App/Sources/Types.elm +++ b/src/App/Sources/Types.elm @@ -12,6 +12,7 @@ import Tracks.Types exposing (..) type Service = AmazonS3 + | Ipfs type alias SourceData = @@ -79,16 +80,7 @@ type alias IsProcessing = --- Other types - - -type alias Model = - { collection : List Source - , isProcessing : IsProcessing - , newSource : Source - , processingErrors : List ( SourceId, String ) - , timestamp : Date - } +-- Messages type Msg @@ -101,9 +93,27 @@ type Msg | Destroy SourceId -- Forms | SetNewSourceProperty Source String String + | SetNewSourceType String | SubmitNewSourceForm + +-- Model + + +type alias Model = + { collection : List Source + , isProcessing : IsProcessing + , newSource : Source + , processingErrors : List ( SourceId, String ) + , timestamp : Date + } + + + +-- Other Types + + type Page = Index | New diff --git a/src/App/Sources/Utils.elm b/src/App/Sources/Utils.elm index c976c27b..d86bab5b 100644 --- a/src/App/Sources/Utils.elm +++ b/src/App/Sources/Utils.elm @@ -2,6 +2,7 @@ module Sources.Utils exposing (..) import Date import Maybe.Extra as Maybe +import Regex import Sources.Encoding import Sources.Types exposing (..) import Time @@ -46,3 +47,17 @@ setProperSourceId model source = storeSources : List Source -> Cmd TopLevel.Msg storeSources = List.map Sources.Encoding.encode >> Users.Data.storeSources + + + +-- 🌱 + + +isMusicFile : String -> Bool +isMusicFile = + Regex.contains (Regex.regex "\\.(mp3|mp4|m4a)$") + + +selectMusicFiles : List String -> List String +selectMusicFiles = + List.filter isMusicFile diff --git a/src/App/Sources/View.elm b/src/App/Sources/View.elm index 1846a62a..b3b2f86a 100644 --- a/src/App/Sources/View.elm +++ b/src/App/Sources/View.elm @@ -32,6 +32,7 @@ import Styles exposing (Classes(Button, ContentBox, InsulationContent, Intro)) -- Services import Sources.Services.AmazonS3 as AmazonS3 +import Sources.Services.Ipfs as Ipfs -- 🍯 @@ -143,6 +144,7 @@ renderSource index ( source, isProcessing, processingError ) = , span [ cssClass ListActions ] [ -- Processing error + -- case processingError of Just err -> span [ title err ] [ Icons.error_outline colorDerivatives.error 16 ] @@ -151,12 +153,14 @@ renderSource index ( source, isProcessing, processingError ) = text "" -- Is processing + -- , if isProcessing == True then span [ title "Processing …" ] [ Icons.sync colorDerivatives.text 16 ] else text "" -- Delete + -- , a [ title "Remove" , source.id @@ -204,31 +208,61 @@ pageNewForm newSource = [ cssClass ContentBox , onSubmit Sources.SubmitNewSourceForm ] - [ h1 + [ -- Title + -- + h1 [] [ text "Add a new source" ] + + -- Intro + -- , p [ cssClass Styles.Intro ] - [ text """ - Choose a type of source, fill in its credentials and add it. - """ + [ text "Choose a type of source, fill in its credentials and add it." + , br + [] + [] + , strong + [] + [ text """ + Make sure CORS is enabled for IPFS and Amazon S3 repositories. + """ + ] ] + + -- Select the type of the source + -- , label [] [ text "Source type/service" ] , div [ cssClass FormStyles.SelectBox ] [ select - [] - [ option - [ value "amazon-s3" ] - [ text "Amazon S3" ] + [ onInput SetNewSourceType ] + (List.map + (\( typStr, labe ) -> + option + [ selected <| (toString newSource.service) == typStr + , value typStr + ] + [ text labe ] + ) + [ ( "AmazonS3", "Amazon S3" ) + , ( "Ipfs", "IPFS" ) + ] + ) , Icons.expand_more (Color.greyscale 0.325) 20 ] + + -- Source properties + -- , div [] (renderSourceProperties newSource) + + -- Submit button + -- , div [] [ button @@ -278,3 +312,6 @@ renderSourceProperties source = case source.service of AmazonS3 -> List.map (propertyRenderer source) AmazonS3.properties + + Ipfs -> + List.map (propertyRenderer source) Ipfs.properties diff --git a/src/Js/processing.js b/src/Js/processing.js index 7228fd27..478e931f 100644 --- a/src/Js/processing.js +++ b/src/Js/processing.js @@ -7,11 +7,18 @@ function getTags(urlGET, urlHEAD) { const reader = new jsmediatags.Reader(fakeURL); const fileReader = new XhrFileReader(fakeURL); const makeXHRRequest = fileReader._makeXHRRequest; + const setRequestHeader = fileReader._setRequestHeader; fileReader._createXHRObject = function() { return new XMLHttpRequest(); }; + fileReader._setRequestHeader = function(xhr, headerName, headerValue) { + if (headerName != "If-Modified-Since") { + setRequestHeader(xhr, headerName, headerValue); + } + }; + fileReader._makeXHRRequest = function(method, ...args) { this._url = method.toUpperCase() === "HEAD" ? urlHEAD : urlGET; return makeXHRRequest.call(this, method, ...args);