- Removed 'txt init'

Format

- New B32 ID

Index

- New option: txt index --print
- Move scheme to peers
- Replace peer.*.conf files with index packed locations
  Instead of adding a URL to peers.*.conf, run `txt pull <url>`

Conversion

- Rewritten converters
- txt-convert looks for a .convert.conf containing `key: value` lines.
- Specifiable topic-roots from .convert.conf.
- Added `Topics:` key, with comma seperated topics.
	If set only those topics will appear in the main index and used as topic roots.
	Other topics will have sub-indices generated, but won't be listed in the main index.
- HTML converter header & footer options
- HTML-index renamed to HTM-index

Internal

- Change types: uuid:Uuid -> id:string
- File_store merges identical texts
- Use peer ID for store path, store peers' texts in .local/share/texts
- Simple URN resolution for converter
	Continue to next feed if parsing one fails
- Phasing-out Archive, replaced by improved packs
- Eliminate Bos, Cohttp, lwt, uri, tls, Re, Ptime, dependencies
- Lock version for Cmdliner, fix dune-project
- Optional resursive store
- Improve header_pack
- Fix recursive mkdir

git-svn-id: file:///srv/svn/repo/kosuzu/trunk@3 eb64cd80-c68d-6f47-b6a3-0ada418499da

lib/header_pack.ml

@@ -1,54 +1,57 @@
-type info_t = { version: int; name: string; archivists: string list }
-type text_t = { id: Msgpck.t; time: Msgpck.t; title: Msgpck.t; authors: Msgpck.t }
-type t = { info: info_t; fields: string list; texts: Msgpck.t; peers: Msgpck.t }
+let version = 0
+type info_t = { version: int; id: string; title: string; people: string list; locations: string list }
+type t = { info: info_t; fields: Msgpck.t; texts: Msgpck.t; peers: Msgpck.t }
 
-let of_id id = Msgpck.Bytes (Id.to_bytes id)
-let to_id pck_id = Id.of_bytes Msgpck.(to_bytes pck_id)
+let of_id id = Msgpck.of_string id
+let to_id = Msgpck.to_string
 
 let person p = Msgpck.String (Person.to_string p)
-let persons ps = List.rev @@ Person.Set.fold (fun p a -> person p :: a) ps []
+let persons ps = Msgpck.of_list @@ List.rev @@ Person.Set.fold (fun p a -> person p :: a) ps []
+
+let str = Msgpck.of_string
+let str_list ls = Msgpck.of_list @@ List.map str ls
+let to_str_list x = List.map Msgpck.to_string (Msgpck.to_list x)
 
 let of_set field t =
 	List.rev @@ String_set.fold (fun s a -> Msgpck.String s :: a) (Text.set field t) []
 
-let date = function
-	| None -> Int32.zero
-	| Some date ->
-		let days, ps = Ptime.Span.to_d_ps (Ptime.to_span date) in
-		Int32.add Int32.(mul (of_int days) 86400l) Int64.(to_int32 (div ps 1000000000000L))
+let date = function "" -> Int32.zero | date -> Int32.of_int (Date.to_secs date)
 
-let to_sec = function
-	Msgpck.Int i -> Int32.of_int i | Msgpck.Uint32 i -> i | x -> Msgpck.to_uint32 x
-
-let public_peers () =
-	Peers.fold_file (fun x a -> Msgpck.String x :: a) [] Peers.public_fname
+let to_sec = function Msgpck.Int i -> Int32.of_int i | Msgpck.Uint32 i -> i | x -> Msgpck.to_uint32 x
 
 let fields = Msgpck.(List [String "id"; String "time";  String "title"; String "authors"; String "topics"])
 let to_fields fieldpack = List.map Msgpck.to_string (Msgpck.to_list fieldpack)
 
-let to_pack a t =
-	let open Text in
-	Msgpck.(List [
-		Bytes (Id.to_bytes t.uuid); of_uint32 (date (Date.listing t.date));
-		String t.title; List (persons t.authors); List (of_set "topics" t)
-	]) :: a
-
-let pack_filename ?(filename="index.pck") archive =
-	let dir = Store.KV.find "Export-Dir" archive.File_store.kv in (*raises Not_found*)
-	dir ^ "/" ^ filename
-
 let to_info = function
-	| Msgpck.List (v::n::a::[]) ->
-		let archivists = List.map Msgpck.to_string (Msgpck.to_list a) in
-		Msgpck.({version = to_int v; name = to_string n; archivists})
+	| Msgpck.List (v::id::n::a::ls::[]) ->
+		let people = to_str_list a in
+		let locations = to_str_list ls in
+		Msgpck.({version = to_int v; id = to_string id; title = to_string n; people; locations})
 	| _ -> invalid_arg "Pack header"
 
+let of_info i = let open Msgpck in
+	List [Int i.version; String i.id; String i.title; str_list i.people; str_list i.locations]
+
+let of_text a t =
+	let open Text in
+	Msgpck.(List [
+		of_id t.id; of_uint32 (date (Date.listing t.date));
+		String t.title; persons t.authors; List (of_set "topics" t)
+	]) :: a
+
+let of_text_list l = Msgpck.List l
+
+let pack p = Msgpck.List [of_info p.info; p.fields; p.texts; p.peers]
+let string p = Bytes.to_string @@ Msgpck.Bytes.to_string @@ pack p
+
 let unpack = function
-	| Msgpck.List (i::f::texts::[]) ->
-		Some { info = to_info i; fields = to_fields f; texts; peers = Msgpck.List [] }
-	| Msgpck.List (i::f::texts::peers::[]) ->
-		Some { info = to_info i; fields = to_fields f; texts; peers }
-	| _ -> None
+	| Msgpck.List (i::fields::texts::[]) ->
+		Ok { info = to_info i; fields; texts; peers = Msgpck.List [] }
+	| Msgpck.List (i::fields::texts::peers::[]) ->
+		Ok { info = to_info i; fields; texts; peers }
+	| _ -> Error "format mismatch"
+
+let of_string s = unpack @@ snd @@ Msgpck.StringBuf.read s
 
 let list filename = try
 	let texts_list = function
@@ -60,25 +63,22 @@ let list filename = try
 
 let contains text = function
 	| Msgpck.List (id::_time::title::_authors::_topics::[]) ->
-		(match Id.of_bytes (Msgpck.to_bytes id) with
-		 | None -> prerr_endline ("Invalid id for " ^ Msgpck.to_string title); false
-		 | Some id -> text.Text.uuid = id)
+		(match to_id id with
+		 | "" -> prerr_endline ("Invalid id for " ^ Msgpck.to_string title); false
+		 | id -> text.Text.id = id)
 	| _ -> prerr_endline ("Invalid record pattern"); false
 
-let pack archive records =
-	let header_pack = List.fold_left to_pack [] records in
-	let info = Msgpck.(List [Int 0; String archive.File_store.name; List (persons archive.archivists)]) in
-	Bytes.to_string @@ Msgpck.Bytes.to_string
-		(List [info; fields; Msgpck.List header_pack; Msgpck.List (public_peers ())])
 
-let add archive records =
-	let fname = pack_filename archive in
-	let append published (t, _f) = if List.exists (contains t) published then published else to_pack published t in
-	match list fname with Error e -> prerr_endline e | Ok published_list ->
-		let header_pack = List.fold_left append published_list records in
-		let archive = Msgpck.(List [Int 0; String archive.File_store.name;
-			List (persons archive.archivists)]) in
-		File_store.file fname @@ Bytes.to_string
-			@@ Msgpck.Bytes.to_string (List [archive; fields; Msgpck.List header_pack])
+(*let pack_filename ?(filename="index.pck") archive =*)
+(*	let dir = Store.KV.find "Export-Dir" archive.File_store.kv in (*raises Not_found*)*)
+(*	dir ^ "/" ^ filename*)
 
-let unpublish _archive _records = ()
+(*let add archive records =*)
+(*	let fname = pack_filename archive in*)
+(*	let append published (t, _f) = if List.exists (contains t) published then published else to_pack published t in*)
+(*	match list fname with Error e -> prerr_endline e | Ok published_list ->*)
+(*		let header_pack = List.fold_left append published_list records in*)
+(*		let archive = Msgpck.(List [*)
+(*			Int 0; String archive.File_store.name; persons archive.people]) in*)
+(*		File_store.file fname @@ Bytes.to_string*)
+(*			@@ Msgpck.Bytes.to_string (List [archive; fields; Msgpck.List header_pack])*)