Skip to content

B
BDIO.jl
Commit 9475449a authored by Alberto Ramos's avatar Alberto Ramos
Browse files
Options

Modular structure. Updated documentation

parent dfd086fb
Hide whitespace changes
Inline Side-by-side
Showing with 523 additions and 590 deletions
docs/make.jl
View file @ 9475449a
using Documenter, BDIO using Documenter, BDIO
makedocs(modules=[BDIO], doctest=true) makedocs(modules=[BDIO], doctest=true, sitename = "BDIO meets Julia")
deploydocs(deps = Deps.pip("mkdocs", "python-markdown-math"), deploydocs(deps = Deps.pip("mkdocs", "python-markdown-math"),
repo = "github.com/GITHUBNAME/GITHUBREPO.git", repo = "github.com/GITHUBNAME/GITHUBREPO.git",
......
docs/src/index.md
View file @ 9475449a
# BDIO in Julia # BDIO in Julia
This package provides an interface to read/write This package provides an interface to read/write
[BDIO](http://bdio.org/) files in Julia. [BDIO](http://bdio.org/) (**B**inary **D**ata **I**nput/**O**utput)
files in Julia.
## Index
```@index
```
## Externally callable functions ## Externally callable functions
### Setting up global information
```@docs
BDIO.BDIO_set_user
BDIO.BDIO_set_host
```
### Openning `BDIO` files ### Openning `BDIO` files
```@docs ```@docs
...@@ -34,7 +47,3 @@ BDIO.BDIO_get_len ...@@ -34,7 +47,3 @@ BDIO.BDIO_get_len
BDIO.BDIO_get_fmt BDIO.BDIO_get_fmt
``` ```
## Index
```@index
```
src/BDIO.jl
View file @ 9475449a
module BDIO module BDIO
import Nettle import Nettle
# Include constant values and global variables
include("BDIOvalues.jl")
# Include types
include("BDIOtypes.jl")
# Include methods
include("BDIOmethods.jl")
export BDIO_BIN_GENERIC, BDIO_ASC_EXEC, BDIO_BIN_INT32BE, export BDIO_BIN_GENERIC, BDIO_ASC_EXEC, BDIO_BIN_INT32BE,
BDIO_BIN_INT32LE, BDIO_BIN_INT64BE, BDIO_BIN_INT64LE, BDIO_BIN_INT32LE, BDIO_BIN_INT64BE, BDIO_BIN_INT64LE,
BDIO_BIN_F32BE, BDIO_BIN_F32LE, BDIO_BIN_F64BE, BDIO_BIN_F64LE, BDIO_BIN_F32BE, BDIO_BIN_F32LE, BDIO_BIN_F64BE, BDIO_BIN_F64LE,
...@@ -11,588 +22,4 @@ export BDIO, BDIO_open, BDIO_start_record!, BDIO_write_hash!, ...@@ -11,588 +22,4 @@ export BDIO, BDIO_open, BDIO_start_record!, BDIO_write_hash!,
BDIO_write!, BDIO_seek!, BDIO_get_len, BDIO_get_fmt, BDIO_write!, BDIO_seek!, BDIO_get_len, BDIO_get_fmt,
BDIO_get_uinfo, BDIO_read BDIO_get_uinfo, BDIO_read
const BDIO_R_MODE = 0
const BDIO_W_MODE = 1
const BDIO_A_MODE = 2
const BDIO_D_MODE = 3
const MAXFNAME = 4096
const BDIO_SHORT_LEN = 256
const BDIO_LONG_LEN = 4096
const BDIO_MAGIC = 2147209342
const BDIO_VERSION = 1
const BDIO_R_STATE = 0
const BDIO_W_STATE = 1
const BDIO_BIN_GENERIC = 0
const BDIO_ASC_EXEC = 1
const BDIO_BIN_INT32BE = 2
const BDIO_BIN_INT32LE = 3
const BDIO_BIN_INT64BE = 4
const BDIO_BIN_INT64LE = 5
const BDIO_BIN_F32BE = 6
const BDIO_BIN_F32LE = 7
const BDIO_BIN_F64BE = 8
const BDIO_BIN_F64LE = 9
const BDIO_ASC_GENERIC = 10
const BDIO_ASC_XML = 11
const BDIO_BIN_INT32 = 240
const BDIO_BIN_INT64 = 241
const BDIO_BIN_F32 = 242
const BDIO_BIN_F64 = 243
const BDIO_HASH_MAGIC_S = 1515784845
user = " "
host = " "
mutable struct Record
ishdr::Bool
islong::Bool
# Record length, start, end
rlen::Int64
rpos::Int64
rend::Int64
rfmt::Int32
ruinfo::Int8
hsh::Nettle.Hasher
end
mutable struct BDIOstream
io::IO
ipt::Int64
imode::Int32
istate::Int32
rwpos::Int64
lendian::Bool
user::String
host::String
created::Int32
modified::Int32
info::String
rdate::Int64
records::Array{Record,1}
end
"""
Set user name globally for writing BDIO files.
### Example
BDIO_set_user("alberto")
"""
function BDIO_set_user(us::String)
global user = SubString(us*user, 1, 255)
return true
end
"""
Set host machine globally for writing BDIO files.
### Example
BDIO_set_host("HLRN")
"""
function BDIO_set_host(us::String)
global host = SubString(us*host, 1, 255)
return true
end
function BDIOstream(fname::String, mode::String, protocol_info::String="\0")
md::Int32 = 0
if (mode == "w")
if (isfile(fname))
error("File "*strip(fname)*" already exists")
end
io = open(fname, "w+")
md = BDIO_W_MODE
elseif (mode == "r")
io = open(fname, "r")
md = BDIO_R_MODE
elseif (mode == "a")
io = open(fname, "a+")
md = BDIO_A_MODE
elseif (mode == "d")
rm(fnme, force=true)
io = open(fname, "w+")
md = BDIO_D_MODE
else
error("Incorrect mode")
end
rcs = similar(Array{Record,1},0)
lend::Bool = true
if (Base.ENDIAN_BOM == 0x04030201)
lend = true
elseif (Base.ENDIAN_BOM == 0x01020304)
lend = false
else
error("Wrong endianness")
end
if (!lend)
error("Current version only support Little Endian Machines")
end
return BDIOstream(io,1,md,0,0,lend,user,host,
floor(Int32, time()), floor(Int32, time()), protocol_info, 0, rcs)
end
"""
Opens a BDIO file and returns the BDIO handle. The file can be opened in several modes:
- Write mode ("w"): The file is created and a header written. If the file exists an error is printed.
- Write mode ("d"): The file is created and a header written. If the file exists it is overwritten.
- Write mode ("a"): The file is created if it does not exist, or opened for appending if the file exists.
- Write mode ("r"): The file is opened for reading.
## Arguments
- `fname`: File name
- `mode`: The mode in which the file is opened. See above.
- `protocol_info`: Only used when the file is created (i.e. "w" mode) and labels the file.
## Examples
```julia-repl
julia> fb = BDIO_open("new_file.bdio", "w", "Test file")
```
## Returns
A BDIOstream type.
"""
function BDIO_open(fname::String, mode::String, protocol_info::String="")
fb = BDIOstream(fname,mode)
fb.info = protocol_info
fb.user = user
fb.host = host
if (mode == "w")
fb.created = floor(Int32, time())
fb.modified = floor(Int32, time())
BDIO_write_header!(fb)
elseif (mode == "r")
BDIO_parse!(fb)
fb.ipt = 1
elseif (mode == "a")
BDIO_parse!(fb)
fb.ipt = length(fb.records)
elseif (mode == "d")
BDIO_parse!(fb)
fb.ipt = length(fb.records)
else
error("Incorrect mode")
end
return fb
end
function BDIO_write_header!(fb::BDIOstream)
if (fb.imode == BDIO_R_MODE)
error("Attemp to write in READ mode")
end
ihdr::Int32 = BDIO_MAGIC
write(fb.io, ihdr)
mark(fb.io)
ill::Int16 = 0
write(fb.io, ill)
iv::Int16 = 1
write(fb.io, iv)
ist::Int64 = position(fb.io)
i4::Int32 = 0
write(fb.io, i4)
hcr::Int64 = position(fb.io)
write(fb.io, fb.created)
fb.rdate = position(fb.io)
write(fb.io, fb.modified)
write(fb.io, user*"\0")
hlu::Int64 = position(fb.io)
write(fb.io, user*"\0")
write(fb.io, host*"\0")
hlh::Int64 = position(fb.io)
write(fb.io, host*"\0")
write(fb.io, fb.info*"\0")
ind::Int64 = position(fb.io)
ill = Int16(ind-ist)
reset(fb.io)
write(fb.io, ill)
flush(fb.io)
hsh = Nettle.Hasher("md5")
new = Record(true,false,Int64(ill),ist,ind,0,0,hsh)
push!(fb.records, new)
fb.ipt += 1
return true
end
"""
Start a new BDIO record at the end of the file. Currently the supported formats are
- `BDIO_BIN_GENERIC`: Generic binary data
- `BDIO_ASC_EXEC`: ASCII Executable (i.e. `sh` script)
- `BDIO_BIN_INT32BE`: 32-bit integer in Big Endian format
- `BDIO_BIN_INT32LE`: 32-bit integer in Little Endian format
- `BDIO_BIN_INT64BE`: 64-bit integer in Big Endian format
- `BDIO_BIN_INT64LE`: 64-bit integer in Little Endian format
- `BDIO_BIN_F32BE`: 32-bit float in Big Endian format
- `BDIO_BIN_F32LE`: 32-bit float in Little Endian format
- `BDIO_BIN_F64BE`: 64-bit float in Big Endian format
- `BDIO_BIN_F64LE`: 64-bit float in Little Endian format
- `BDIO_ASC_GENERIC`: ASCII text file
- `BDIO_ASC_XML`: Plain XML data
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file in either `w` or `a` mode
- `ifmt`: Format (see above).
- `iuinfo`: Integer in the range 0-15. A user specified label to help indentifying the record
- `long` (optional): If true create a long record. To store more than 1048575 bytes of data (``\\approx 1\\, {\\rm MB}``), a long record is required. Default value of `false`.
## Examples
```julia-repl
julia> fb = BDIO_open("new_file.bdio", "w", "Test file")
julia> BDIO_start_record!(fb, BDIO.BDIO_BIN_F32, 2)
```
"""
function BDIO_start_record!(fb::BDIOstream, ifmt, iuinfo, long::Bool = false)
if (fb.imode == BDIO_R_MODE)
error("Attemp to write in READ mode")
end
seekend(fb.io)
shdr::Int32 = 0
if long
shdr = 0b1001
else
shdr = 0b1
end
shdr = shdr | ifmt<<4
shdr = shdr | iuinfo<<8
write(fb.io,shdr)
if long
zero::Int32 = 0
write(fb.io,zero)
end
ist::Int64 = position(fb.io)
hsh = Nettle.Hasher("md5")
new = Record(false,long,Int64(0),ist,ist,Int32(ifmt),Int8(iuinfo),hsh)
push!(fb.records, new)
fb.ipt += 1
flush(fb.io)
return true
end
"""
Write `data` to BDIO file to the end of the last record.
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file in either `w` or `a` mode.
- `data`: Data to write to file.
- `hash` (optional): Bool. If true, data will be checksummed before writtento file. The hash can be stored in a BDIO record by calling `BDIO_write_hash(...)`
## Examples
```julia-repl
julia> # Write 1000 normal random numbers to file `randoms.bdio`
julia> fb = BDIO_open("randoms.bdio", "w", "File with random numbers")
julia> BDIO_start_record!(fb, BDIO.BDIO_BIN_F32, 2)
julia> BDIO_write!(fb, randn(1000))
```
"""
function BDIO_write!(fb,data,hash::Bool=true)
if (fb.imode == BDIO_R_MODE)
error("Attemp to write in READ mode")
end
ir = length(fb.records)
fb.records[ir].rlen += sizeof(data)
fb.records[ir].rend += sizeof(data)
update_length(fb, ir)
seekend(fb.io)
write(fb.io, data)
flush(fb.io)
if hash
Nettle.update!(fb.records[ir].hsh, data)
end
update_hdrinfo!(fb)
return true
end
"""
Write the `MD5` checksum of the actual record as a new record.
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file in either `w` or `a` mode.
## Examples
```julia-repl
julia> # Write 1000 normal random numbers to file `randoms.bdio` with checksum
julia> fb = BDIO_open("randoms.bdio", "w", "File with random numbers")
julia> BDIO_start_record!(fb, BDIO.BDIO_BIN_F32, 2)
julia> BDIO_write!(fb, randn(1000))
julia> BDIO_write_hash!(fb)
```
"""
function BDIO_write_hash!(fb)
if (fb.imode == BDIO_R_MODE)
error("Attemp to write in READ mode")
end
ir = length(fb.records)
BDIO_start_record!(fb,BDIO_BIN_GENERIC,7)
i4::Int32 = BDIO_HASH_MAGIC_S
BDIO_write!(fb, i4, false)
BDIO_write!(fb, Nettle.digest!(fb.records[ir].hsh))
end
function update_length(fb::BDIOstream, ir)
mark(fb.io)
if fb.records[ir].islong
seek(fb.io,fb.records[ir].rpos-8)
r64 = read(fb.io,Int64)
mask64::Int64 = 0b111111111111
mask64 = mask64 & r64
il64::Int64 = (fb.records[ir].rend - fb.records[ir].rpos)<<12
iw64::Int64 = il64 | mask64
seek(fb.io,fb.records[ir].rpos-8)
write(fb.io,iw64)
else
seek(fb.io,fb.records[ir].rpos-4)
r32 = read(fb.io,Int32)
mask32::Int32 = 0b111111111111
mask32 = mask32 & r32
il32::Int32 = Int32((fb.records[ir].rend - fb.records[ir].rpos))<<12
iw32::Int32 = il32 | mask32
seek(fb.io,fb.records[ir].rpos-4)
write(fb.io,iw32)
end
flush(fb.io)
reset(fb.io)
return
end
function update_hdrinfo!(fb)
mark(fb.io)
seek(fb.io, fb.rdate)
write(fb.io,floor(Int32, time()))
flush(fb.io)
reset(fb.io)
end
function BDIO_parse!(fb::BDIOstream)
seekstart(fb.io)
ihdr = read(fb.io, Int32)
mask32::Int32 = 0b111111111111
rlen = Int64(read(fb.io, Int32) & mask32)
rpos = position(fb.io)
hsh = Nettle.Hasher("md5")
il4 = read(fb.io, Int32)
idt1 = read(fb.io, Int32)
idt2 = read(fb.io, Int32)
user = readuntil(fb.io, '\0')
user = readuntil(fb.io, '\0')
host = readuntil(fb.io, '\0')
host = readuntil(fb.io, '\0')
info = readuntil(fb.io, '\0')
if (ihdr!=BDIO_MAGIC)
error("Not a BDIO file")
end
new = Record(true,false,rlen,rpos,rpos+rlen,0,0,hsh)
push!(fb.records, new)
seek(fb.io, rpos+rlen)
rlen::Int64 = 0
rpos::Int64 = 0
while !eof(fb.io)
ihdr = read(fb.io, Int32)
ishdr = (mask = 1<<0; ihdr & mask != mask)
if ishdr
if (ihdr != BDIO_MAGIC)
error("Not a BDIO file")
end
i32::Int32 = read(fb.io, Int32)
rlen = Int64(i32 & 0b111111111111)
rpos = position(fb.io)
new = Record(true,false,rlen,rpos,rpos+rlen,0,0,hsh)
else
islong = (mask = 1<<3; ihdr & mask == mask)
ifmt::Int32 = (ihdr & 0b11110000)>>>4
iuinfo::Int8 = Int8((ihdr & 0b111100000000)>>>8)
if islong
rpos = position(fb.io) + 4
ihdr2 = read(fb.io, Int32)
rlen = ihdr>>>12 | ihdr2<<32
else
rpos = position(fb.io)
rlen = ihdr>>>12
end
new = Record(false,islong,rlen,rpos,rpos+rlen,Int32(ifmt),Int8(iuinfo),hsh)
end
push!(fb.records, new)
skip(fb.io, rlen)
end
fb.ipt = 1
end
"""
Move the read position backward/forward `icnt` records
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file.
- `icnt` (optional): number of records to move forward (if `icnt>0`) or backwards (`icnt<0`). The default value is `+1`.
## Examples
```julia-repl
julia> # count number of records in a file.
julia> fb = BDIO_open("randoms.bdio", "r")
julia> count = 0
julia> while BDIO_seek(fb)
julia> count += 1
julia> end
```
"""
function BDIO_seek!(fb::BDIOstream, icnt::Int = 1)
ipt::Int64 = fb.ipt
if (icnt == 0)
ipt = 1
else
ipt += icnt
end
if (ipt < length(fb.records))&&(ipt > 0)
fb.ipt = ipt
return true
else
return false
end
end
"""
Read `data` from BDIO file.
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file in either `w` or `a` mode.
- `data[:]`: A `Vector` of data to read.
- `n` (optional): Integer. If present read `n` elements of `data[:]` from file
## Examples
```julia-repl
julia> # Real 1000 floats from the first record of file `randoms.bdio`
julia> fb = BDIO_open("randoms.bdio", "r")
julia> BDIO_seek!(fb)
julia> data = similar({Array, 1}, 1000)
julia> BDIO_read(fb, data)
```
"""
function BDIO_read(fb, vdata::Vector, n::Int64 = 0)
if (n>0)
nmax = n
else
nmax = length(vdata)
end
if (nmax*sizeof(vdata[1]) > fb.records[fb.ipt].rlen)
error("Data in record not available")
end
seek(fb.io,fb.records[fb.ipt].rpos)
for i = 1:nmax
vdata[i] = read(fb.io, typeof(vdata[1]))
end
end
"""
Returns the len (in bytes) of the current record
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file.
## Examples
```julia-repl
julia> # Write length of all records
julia> fb = BDIO_open("randoms.bdio", "r")
julia> while BDIO_seek(fb)
julia> count += 1
julia> println("Record: ", count, " length: ", BDIO_get_len(fb), " bytes")
julia> end
```
"""
BDIO_get_len(fb::BDIOstream) = fb.records[fb.ipt].rlen
"""
Returns the format of the current record
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file.
## Examples
```julia-repl
julia> # Print all BDIO_BIN_GENERIC records
julia> fb = BDIO_open("randoms.bdio", "r")
julia> while BDIO_seek(fb)
julia> count += 1
julia> if BDIO_get_fmt(fb)
julia> println("Record: ", count)
julia> end
julia> end
```
"""
BDIO_get_fmt(fb::BDIOstream) = fb.records[fb.ipt].rfmt
"""
Returns the user provided info of each record
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file.
## Examples
```julia-repl
julia> # Write length of all records
julia> fb = BDIO_open("randoms.bdio", "r")
julia> while BDIO_seek(fb)
julia> count += 1
julia> println("Record: ", count, " user info: ", BDIO_get_uinfo(fb))
julia> end
```
"""
BDIO_get_uinfo(fb::BDIOstream) = fb.records[fb.ipt].ruinfo
end # module end # module
src/BDIOmethods.jl 0 → 100644
View file @ 9475449a
"""
Set user name globally for writing BDIO files.
### Example
BDIO_set_user("alberto")
"""
function BDIO_set_user(us::String)
global user = SubString(us*user, 1, 255)
return true
end
"""
Set host machine globally for writing BDIO files.
### Example
BDIO_set_host("HLRN")
"""
function BDIO_set_host(us::String)
global host = SubString(us*host, 1, 255)
return true
end
"""
Opens a BDIO file and returns the BDIO handle. The file can be opened in several modes:
- Write mode ("w"): The file is created and a header written. If the file exists an error is printed.
- Write mode ("d"): The file is created and a header written. If the file exists it is overwritten.
- Write mode ("a"): The file is created if it does not exist, or opened for appending if the file exists.
- Write mode ("r"): The file is opened for reading.
## Arguments
- `fname`: File name
- `mode`: The mode in which the file is opened. See above.
- `protocol_info`: Only used when the file is created (i.e. "w" mode) and labels the file.
## Examples
```julia-repl
julia> fb = BDIO_open("new_file.bdio", "w", "Test file")
```
## Returns
A BDIOstream type.
"""
function BDIO_open(fname::String, mode::String, protocol_info::String="")
fb = BDIOstream(fname,mode)
fb.info = protocol_info
fb.user = user
fb.host = host
if (mode == "w")
fb.created = floor(Int32, time())
fb.modified = floor(Int32, time())
BDIO_write_header!(fb)
elseif (mode == "r")
BDIO_parse!(fb)
fb.ipt = 1
elseif (mode == "a")
BDIO_parse!(fb)
fb.ipt = length(fb.records)
elseif (mode == "d")
BDIO_parse!(fb)
fb.ipt = length(fb.records)
else
error("Incorrect mode")
end
return fb
end
"""
Start a new BDIO record at the end of the file. Currently the supported formats are
- `BDIO_BIN_GENERIC`: Generic binary data
- `BDIO_ASC_EXEC`: ASCII Executable (i.e. `sh` script)
- `BDIO_BIN_INT32BE`: 32-bit integer in Big Endian format
- `BDIO_BIN_INT32LE`: 32-bit integer in Little Endian format
- `BDIO_BIN_INT64BE`: 64-bit integer in Big Endian format
- `BDIO_BIN_INT64LE`: 64-bit integer in Little Endian format
- `BDIO_BIN_F32BE`: 32-bit float in Big Endian format
- `BDIO_BIN_F32LE`: 32-bit float in Little Endian format
- `BDIO_BIN_F64BE`: 64-bit float in Big Endian format
- `BDIO_BIN_F64LE`: 64-bit float in Little Endian format
- `BDIO_ASC_GENERIC`: ASCII text file
- `BDIO_ASC_XML`: Plain XML data
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file in either `w` or `a` mode
- `ifmt`: Format (see above).
- `iuinfo`: Integer in the range 0-15. A user specified label to help indentifying the record
- `long` (optional): If true create a long record. To store more than 1048575 bytes of data (``\\approx 1\\, {\\rm MB}``), a long record is required. Default value of `false`.
## Examples
```julia-repl
julia> fb = BDIO_open("new_file.bdio", "w", "Test file")
julia> BDIO_start_record!(fb, BDIO.BDIO_BIN_F32, 2)
```
"""
function BDIO_start_record!(fb::BDIOstream, ifmt, iuinfo, long::Bool = false)
if (fb.imode == BDIO_R_MODE)
error("Attemp to write in READ mode")
end
seekend(fb.io)
shdr::Int32 = 0
if long
shdr = 0b1001
else
shdr = 0b1
end
shdr = shdr | ifmt<<4
shdr = shdr | iuinfo<<8
write(fb.io,shdr)
if long
zero::Int32 = 0
write(fb.io,zero)
end
ist::Int64 = position(fb.io)
hsh = Nettle.Hasher("md5")
new = Record(false,long,Int64(0),ist,ist,Int32(ifmt),Int8(iuinfo),hsh)
push!(fb.records, new)
fb.ipt += 1
flush(fb.io)
return true
end
"""
Write `data` to BDIO file to the end of the last record.
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file in either `w` or `a` mode.
- `data`: Data to write to file.
- `hash` (optional): Bool. If true, data will be checksummed before writtento file. The hash can be stored in a BDIO record by calling `BDIO_write_hash(...)`
## Examples
```julia-repl
julia> # Write 1000 normal random numbers to file `randoms.bdio`
julia> fb = BDIO_open("randoms.bdio", "w", "File with random numbers")
julia> BDIO_start_record!(fb, BDIO.BDIO_BIN_F32, 2)
julia> BDIO_write!(fb, randn(1000))
```
"""
function BDIO_write!(fb,data,hash::Bool=true)
if (fb.imode == BDIO_R_MODE)
error("Attemp to write in READ mode")
end
ir = length(fb.records)
fb.records[ir].rlen += sizeof(data)
fb.records[ir].rend += sizeof(data)
update_length(fb, ir)
seekend(fb.io)
write(fb.io, data)
flush(fb.io)
if hash
Nettle.update!(fb.records[ir].hsh, data)
end
update_hdrinfo!(fb)
return true
end
"""
Write the `MD5` checksum of the actual record as a new record.
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file in either `w` or `a` mode.
## Examples
```julia-repl
julia> # Write 1000 normal random numbers to file `randoms.bdio` with checksum
julia> fb = BDIO_open("randoms.bdio", "w", "File with random numbers")
julia> BDIO_start_record!(fb, BDIO.BDIO_BIN_F32, 2)
julia> BDIO_write!(fb, randn(1000))
julia> BDIO_write_hash!(fb)
```
"""
function BDIO_write_hash!(fb)
if (fb.imode == BDIO_R_MODE)
error("Attemp to write in READ mode")
end
ir = length(fb.records)
BDIO_start_record!(fb,BDIO_BIN_GENERIC,7)
i4::Int32 = BDIO_HASH_MAGIC_S
BDIO_write!(fb, i4, false)
BDIO_write!(fb, Nettle.digest!(fb.records[ir].hsh))
end
"""
Move the read position backward/forward `icnt` records
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file.
- `icnt` (optional): number of records to move forward (if `icnt>0`) or backwards (`icnt<0`). The default value is `+1`.
## Examples
```julia-repl
julia> # count number of records in a file.
julia> fb = BDIO_open("randoms.bdio", "r")
julia> count = 0
julia> while BDIO_seek(fb)
julia> count += 1
julia> end
```
"""
function BDIO_seek!(fb::BDIOstream, icnt::Int = 1)
ipt::Int64 = fb.ipt
if (icnt == 0)
ipt = 1
else
ipt += icnt
end
if (ipt < length(fb.records))&&(ipt > 0)
fb.ipt = ipt
return true
else
return false
end
end
"""
Read `data` from BDIO file.
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file in either `w` or `a` mode.
- `data[:]`: A `Vector` of data to read.
- `n` (optional): Integer. If present read `n` elements of `data[:]` from file
## Examples
```julia-repl
julia> # Real 1000 floats from the first record of file `randoms.bdio`
julia> fb = BDIO_open("randoms.bdio", "r")
julia> BDIO_seek!(fb)
julia> data = similar({Array, 1}, 1000)
julia> BDIO_read(fb, data)
```
"""
function BDIO_read(fb, vdata::Vector, n::Int64 = 0)
if (n>0)
nmax = n
else
nmax = length(vdata)
end
if (nmax*sizeof(vdata[1]) > fb.records[fb.ipt].rlen)
error("Data in record not available")
end
seek(fb.io,fb.records[fb.ipt].rpos)
for i = 1:nmax
vdata[i] = read(fb.io, typeof(vdata[1]))
end
end
"""
Returns the len (in bytes) of the current record
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file.
## Examples
```julia-repl
julia> # Write length of all records
julia> fb = BDIO_open("randoms.bdio", "r")
julia> while BDIO_seek(fb)
julia> count += 1
julia> println("Record: ", count, " length: ", BDIO_get_len(fb), " bytes")
julia> end
```
"""
BDIO_get_len(fb::BDIOstream) = fb.records[fb.ipt].rlen
"""
Returns the format of the current record
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file.
## Examples
```julia-repl
julia> # Print all BDIO_BIN_GENERIC records
julia> fb = BDIO_open("randoms.bdio", "r")
julia> while BDIO_seek(fb)
julia> count += 1
julia> if (BDIO_get_fmt(fb) == BDIO_BIN_GENERIC)
julia> println("Record: ", count, " is BIN_GENERIC")
julia> end
julia> end
```
"""
BDIO_get_fmt(fb::BDIOstream) = fb.records[fb.ipt].rfmt
"""
Returns the user provided info of each record
## Arguments
- `fb`: A BDIOstream type. It must be associated with a file.
## Examples
```julia-repl
julia> # Write length of all records
julia> fb = BDIO_open("randoms.bdio", "r")
julia> while BDIO_seek(fb)
julia> count += 1
julia> println("Record: ", count, " user info: ", BDIO_get_uinfo(fb))
julia> end
```
"""
BDIO_get_uinfo(fb::BDIOstream) = fb.records[fb.ipt].ruinfo
function update_length(fb::BDIOstream, ir)
mark(fb.io)
if fb.records[ir].islong
seek(fb.io,fb.records[ir].rpos-8)
r64 = read(fb.io,Int64)
mask64::Int64 = 0b111111111111
mask64 = mask64 & r64
il64::Int64 = (fb.records[ir].rend - fb.records[ir].rpos)<<12
iw64::Int64 = il64 | mask64
seek(fb.io,fb.records[ir].rpos-8)
write(fb.io,iw64)
else
seek(fb.io,fb.records[ir].rpos-4)
r32 = read(fb.io,Int32)
mask32::Int32 = 0b111111111111
mask32 = mask32 & r32
il32::Int32 = Int32((fb.records[ir].rend - fb.records[ir].rpos))<<12
iw32::Int32 = il32 | mask32
seek(fb.io,fb.records[ir].rpos-4)
write(fb.io,iw32)
end
flush(fb.io)
reset(fb.io)
return
end
function update_hdrinfo!(fb)
mark(fb.io)
seek(fb.io, fb.rdate)
write(fb.io,floor(Int32, time()))
flush(fb.io)
reset(fb.io)
end
function BDIO_parse!(fb::BDIOstream)
seekstart(fb.io)
ihdr = read(fb.io, Int32)
mask32::Int32 = 0b111111111111
rlen = Int64(read(fb.io, Int32) & mask32)
rpos = position(fb.io)
hsh = Nettle.Hasher("md5")
il4 = read(fb.io, Int32)
idt1 = read(fb.io, Int32)
idt2 = read(fb.io, Int32)
user = readuntil(fb.io, '\0')
user = readuntil(fb.io, '\0')
host = readuntil(fb.io, '\0')
host = readuntil(fb.io, '\0')
info = readuntil(fb.io, '\0')
if (ihdr!=BDIO_MAGIC)
error("Not a BDIO file")
end
new = Record(true,false,rlen,rpos,rpos+rlen,0,0,hsh)
push!(fb.records, new)
seek(fb.io, rpos+rlen)
rlen::Int64 = 0
rpos::Int64 = 0
while !eof(fb.io)
ihdr = read(fb.io, Int32)
ishdr = (mask = 1<<0; ihdr & mask != mask)
if ishdr
if (ihdr != BDIO_MAGIC)
error("Not a BDIO file")
end
i32::Int32 = read(fb.io, Int32)
rlen = Int64(i32 & 0b111111111111)
rpos = position(fb.io)
new = Record(true,false,rlen,rpos,rpos+rlen,0,0,hsh)
else
islong = (mask = 1<<3; ihdr & mask == mask)
ifmt::Int32 = (ihdr & 0b11110000)>>>4
iuinfo::Int8 = Int8((ihdr & 0b111100000000)>>>8)
if islong
rpos = position(fb.io) + 4
ihdr2 = read(fb.io, Int32)
rlen = ihdr>>>12 | ihdr2<<32
else
rpos = position(fb.io)
rlen = ihdr>>>12
end
new = Record(false,islong,rlen,rpos,rpos+rlen,Int32(ifmt),Int8(iuinfo),hsh)
end
push!(fb.records, new)
skip(fb.io, rlen)
end
fb.ipt = 1
end
src/BDIOtypes.jl 0 → 100644
View file @ 9475449a
mutable struct Record
ishdr::Bool
islong::Bool
# Record length, start, end
rlen::Int64
rpos::Int64
rend::Int64
rfmt::Int32
ruinfo::Int8
hsh::Nettle.Hasher
end
mutable struct BDIOstream
io::IO
ipt::Int64
imode::Int32
istate::Int32
rwpos::Int64
lendian::Bool
user::String
host::String
created::Int32
modified::Int32
info::String
rdate::Int64
records::Array{Record,1}
end
src/BDIOvalues.jl 0 → 100644
View file @ 9475449a
const BDIO_R_MODE = 0
const BDIO_W_MODE = 1
const BDIO_A_MODE = 2
const BDIO_D_MODE = 3
const MAXFNAME = 4096
const BDIO_SHORT_LEN = 256
const BDIO_LONG_LEN = 4096
const BDIO_MAGIC = 2147209342
const BDIO_VERSION = 1
const BDIO_R_STATE = 0
const BDIO_W_STATE = 1
const BDIO_BIN_GENERIC = 0
const BDIO_ASC_EXEC = 1
const BDIO_BIN_INT32BE = 2
const BDIO_BIN_INT32LE = 3
const BDIO_BIN_INT64BE = 4
const BDIO_BIN_INT64LE = 5
const BDIO_BIN_F32BE = 6
const BDIO_BIN_F32LE = 7
const BDIO_BIN_F64BE = 8
const BDIO_BIN_F64LE = 9
const BDIO_ASC_GENERIC = 10
const BDIO_ASC_XML = 11
const BDIO_BIN_INT32 = 240
const BDIO_BIN_INT64 = 241
const BDIO_BIN_F32 = 242
const BDIO_BIN_F64 = 243
const BDIO_HASH_MAGIC_S = 1515784845
user = " "
host = " "
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment