Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions DataParser.Console/DataParser.Console.fsproj
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,8 @@
</PropertyGroup>

<ItemGroup>
<Compile Include="TaskBuilder.fs" />
<Compile Include="TaskOperators.fs" />
<Compile Include="Task.fs" />
<Compile Include="Map.fs" />
<Compile Include="Result.fs" />
<Compile Include="ResultBuilder.fs" />
<Compile Include="ResultMap.fs" />
Expand Down
18 changes: 13 additions & 5 deletions DataParser.Console/FileWrite.fs
Original file line number Diff line number Diff line change
@@ -1,20 +1,28 @@
module DataParser.Console.FileWrite

open System
open System.Text.Json
open System.Threading.Tasks
open System.IO
open DataParser.Console.DataFiles

let writeOutputFile folderPath (fileMap : DataFileParseResult) =
let newLineUtf8 = System.Text.Encoding.UTF8.GetBytes Environment.NewLine

let writeOutputFileAsync folderPath (fileMap : DataFileParseResult) =
let serializeElement (JsonObject jsonObject) =
let serialized = JsonSerializer.Serialize jsonObject
System.Text.Encoding.UTF8.GetBytes $"{serialized}\n"
let serialized = JsonSerializer.SerializeToUtf8Bytes jsonObject
Array.concat [| serialized; newLineUtf8 |]

let writeBytes (stream: Stream) (bytes: byte array) = stream.Write bytes
let writeBytesAsync (stream: Stream) (bytes: byte array) =
bytes
|> stream.WriteAsync
|> _.AsTask()

let createOutputFilePath =
(+) folderPath << sprintf "/%s" << formatOutputFileName

ignore <| Directory.CreateDirectory folderPath
let filePath = createOutputFilePath fileMap.DataFileName
use fs = File.Open (filePath, FileMode.Create)
Seq.iter (writeBytes fs << serializeElement) fileMap.JsonElements
let tasks = Seq.map (writeBytesAsync fs << serializeElement) fileMap.JsonElements
Task.WhenAll tasks
10 changes: 0 additions & 10 deletions DataParser.Console/Map.fs

This file was deleted.

18 changes: 8 additions & 10 deletions DataParser.Console/Program.fs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
open System
open System.Threading.Tasks
open DataParser.Console.FileRead
open DataParser.Console.FileWrite
open ResultMap
Expand All @@ -12,26 +13,21 @@ let DataFolderPath = "./data"
[<Literal>]
let OutputFolderPath = "./output"

let okHandler _ = writeOutputFile OutputFolderPath
let okHandler _ = writeOutputFileAsync OutputFolderPath

let errorHandler filePath errors =
eprintfn $"Error occurred during processing data file: {filePath}. Errors are : %+A{errors}"

let consolidateResults (ResultMap dataFileFormats) =
let folder acc k v =
match v with
let folder acc k = function
| Ok dataFileFormat ->
task {
let! parseResult = parseDataFile dataFileFormat
match parseResult with
| Ok result ->
return! Task.liftA3 Map.add (Task.singleton k) (Task.singleton (Ok result)) acc
| Error e ->
return! Task.liftA3 Map.add (Task.singleton k) (Task.singleton (Error e)) acc
return! Map.add <!> Task.singleton k <*> Task.singleton parseResult <*> acc
}
| Error e ->
task {
return! Task.liftA3 Map.add (Task.singleton k) (Task.singleton (Error e)) acc
return! Map.add <!> Task.singleton k <*> Task.singleton (Error e) <*> acc
}

Map.fold folder (Task.singleton Map.empty) dataFileFormats
Expand All @@ -50,8 +46,10 @@ let t =

let! consolidatedResults = consolidateResults dataFileFormats

let result = ResultMap.either okHandler ((<<) Task.fromUnit << errorHandler) consolidatedResults

printfn "Writing to output folder..."
ResultMap.biIter okHandler errorHandler consolidatedResults
do! Task.WhenAll(Map.values result)

printfn "Processing complete. Press Enter to exit."
ignore <| Console.ReadLine()
Expand Down
19 changes: 2 additions & 17 deletions DataParser.Console/ResultMap.fs
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,6 @@ module ResultMap =
ResultMap
<< Map.map (fun _ -> Result.map f)
<< unResultMap

let traverseTask f (ResultMap m) =
let folder acc k v = task {
let! acc' = acc
match v with
| Ok x ->
let! t = f x
return Map.add k (Ok t) acc'
| Error e ->
return Map.add k (Error e) acc'

}

Map.fold folder (task { return Map.empty }) m
|> Task.map ResultMap

let bindResult f =
ResultMap
Expand All @@ -43,6 +28,6 @@ module ResultMap =
let tryFind key (ResultMap m) =
Map.tryFind key m

let biIter f g (ResultMap x) =
let either f g (ResultMap x) =
let go k = function Ok v -> f k v | Error e -> g k e
Map.iter go x
Map.map go x
18 changes: 2 additions & 16 deletions DataParser.Console/Task.fs
Original file line number Diff line number Diff line change
Expand Up @@ -3,28 +3,14 @@ module Task

open System.Threading.Tasks

let map f x = task {
let! result = x
return f result
}

let bind f x = task {
let! result = x
return! f result
}
let map = (<!>)

let toUnit (x: Task) = task {
do! x
return ()
}

let (<!>) = map

let (<*>) (f: Task<'a -> 'b>) (x: Task<'a>) = task {
let tasks = [|f :> Task; x :> Task|]
let! _ = Task.WhenAll(tasks)
return f.Result x.Result
}
let fromUnit (x: unit) = Task.FromResult x :> Task

let liftA3 f x y z = f <!> x <*> y <*> z

Expand Down
20 changes: 0 additions & 20 deletions DataParser.Console/TaskBuilder.fs

This file was deleted.

15 changes: 15 additions & 0 deletions DataParser.Console/TaskOperators.fs
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
[<AutoOpen>]
module TaskOperators

open System.Threading.Tasks

let (<!>) f x = task {
let! result = x
return f result
}

let (<*>) (f: Task<'a -> 'b>) (x: Task<'a>) = task {
let tasks = [|f :> Task; x :> Task|]
let! _ = Task.WhenAll(tasks)
return f.Result x.Result
}