Coder Social home page Coder Social logo

ocaml-cstruct's Introduction

Cstruct - access C-like structures directly from OCaml

Cstruct is a library and syntax extension to make it easier to access C-like structures directly from OCaml. It supports both reading and writing to these structures, and they are accessed via the Bigarray module.

Installation

This repository provides several packages that can be installed via the opam package manager:

  • cstruct: the core Cstruct library
  • cstruct-sexp: serialisers into s-expression format of Cstructs
  • cstruct-unix: provide Unix variations of the read/write functions using file descriptors
  • cstruct-async: provide Async Pipe and Bigstring support
  • cstruct-lwt: provide Lwt variants of read/write functions
  • ppx_cstruct: a PPX syntax extension (see below)

The libraries depend on OCaml version 4.08.0 and later, since it provides a ppx extension point. The old camlp4 syntax extension is nolonger available; the last cstruct release which contained it was v1.9.0.

Local development

You can build the library via dune, using make or dune build directly. Since everything is built via dune, you can also place this repository within a wider dune workspace in order to make local modifications across repositories.

Documentation

A documentation of the last version of cstruct is available here.

Usage

PPX

The PPX processor is used by passing the OCaml source code through the ppx_cstruct binary. An example pcap description is:

[%%cstruct
type pcap_header = {
  magic_number: uint32_t;   (* magic number *)
  version_major: uint16_t;  (* major version number *)
  version_minor: uint16_t;  (* minor version number *)
  thiszone: uint32_t;       (* GMT to local correction *)
  sigfigs: uint32_t;        (* accuracy of timestamps *)
  snaplen: uint32_t;        (* max length of captured packets, in octets *)
  network: uint32_t;        (* data link type *)
} [@@little_endian]]

[%%cstruct
type pcap_packet = {
  ts_sec: uint32_t;         (* timestamp seconds *)
  ts_usec: uint32_t;        (* timestamp microseconds *)
  incl_len: uint32_t;       (* number of octets of packet saved in file *)
  orig_len: uint32_t;       (* actual length of packet *)
} [@@little_endian]]

[%%cstruct
type ethernet = {
  dst: uint8_t [@len 6];
  src: uint8_t [@len 6];
  ethertype: uint16_t;
} [@@big_endian]]

[%%cstruct
type ipv4 = {
  hlen_version: uint8_t;
  tos: uint8_t;
  len: uint16_t;
  id: uint16_t;
  off: uint16_t;
  ttl: uint8_t;
  proto: uint8_t;
  csum: uint16_t;
  src: uint8_t [@len 4];
  dst: uint8_t [@len 4];
} [@@big_endian]]

This auto-generates generates functions of the form below in the ml file:

let sizeof_pcap_packet = 16
let get_pcap_packet_ts_sec v = Cstruct.LE.get_uint32 v 0
let set_pcap_packet_ts_sec v x = Cstruct.LE.set_uint32 v 0 x
let get_pcap_packet_ts_usec v = Cstruct.LE.get_uint32 v 4
let set_pcap_packet_ts_usec v x = Cstruct.LE.set_uint32 v 4 x
let get_pcap_packet_incl_len v = Cstruct.LE.get_uint32 v 8
let set_pcap_packet_incl_len v x = Cstruct.LE.set_uint32 v 8 x
let get_pcap_packet_orig_len v = Cstruct.LE.get_uint32 v 12
let set_pcap_packet_orig_len v x = Cstruct.LE.set_uint32 v 12 x

let sizeof_ethernet = 14
let get_ethernet_dst src = Cstruct.sub src 0 6
let copy_ethernet_dst src = Cstruct.copy src 0 6
let set_ethernet_dst src srcoff dst =
  Cstruct.blit_from_string src srcoff dst 0 6
let blit_ethernet_dst src srcoff dst = Cstruct.blit src srcoff dst 0 6
let get_ethernet_src src = Cstruct.sub src 6 6
let copy_ethernet_src src = Cstruct.copy src 6 6
let set_ethernet_src src srcoff dst =
  Cstruct.blit_from_string src srcoff dst 6 6
let blit_ethernet_src src srcoff dst = Cstruct.blit src srcoff dst 6 6
let get_ethernet_ethertype v = Cstruct.BE.get_uint16 v 12
let set_ethernet_ethertype v x = Cstruct.BE.set_uint16 v 12 x

The mli file will have signatures of this form:

val sizeof_pcap_packet : int
val get_pcap_packet_ts_sec : Cstruct.t -> Cstruct.uint32
val set_pcap_packet_ts_sec : Cstruct.t -> Cstruct.uint32 -> unit
val get_pcap_packet_ts_usec : Cstruct.t -> Cstruct.uint32
val set_pcap_packet_ts_usec : Cstruct.t -> Cstruct.uint32 -> unit
val get_pcap_packet_incl_len : Cstruct.t -> Cstruct.uint32
val set_pcap_packet_incl_len : Cstruct.t -> Cstruct.uint32 -> unit
val get_pcap_packet_orig_len : Cstruct.t -> Cstruct.uint32
val set_pcap_packet_orig_len : Cstruct.t -> Cstruct.uint32 -> unit
val hexdump_pcap_packet_to_buffer : Buffer.t -> pcap_packet -> unit
val hexdump_pcap_packet : Cstruct.t -> unit

val sizeof_ethernet : int
val get_ethernet_dst : Cstruct.t -> Cstruct.t
val copy_ethernet_dst : Cstruct.t -> string
val set_ethernet_dst : string -> int -> Cstruct.t -> unit
val blit_ethernet_dst : Cstruct.t -> int -> Cstruct.t -> unit
val get_ethernet_src : Cstruct.t -> Cstruct.t
val copy_ethernet_src : Cstruct.t -> string
val set_ethernet_src : string -> int -> Cstruct.t -> unit
val blit_ethernet_src : Cstruct.t -> int -> Cstruct.t -> unit
val get_ethernet_ethertype : Cstruct.t -> Cstruct.uint16
val set_ethernet_ethertype : Cstruct.t -> Cstruct.uint16 -> unit
val hexdump_ethernet_to_buffer : Buffer.t -> Cstruct.t -> unit
val hexdump_ethernet : Cstruct.t -> unit

The hexdump functions above are convenient pretty-printing functions to help you debug, and aren't intended to be high performance.

You can also declare C-like enums:

[%%cenum
type foo32 =
  | ONE32
  | TWO32 [@id 0xfffffffel]
  | THREE32
  [@@uint32_t]
]

[%%cenum
type bar16 =
  | ONE [@id 1]
  | TWO
  | FOUR [@id 4]
  | FIVE
  [@@uint16_t]
]

This generates signatures of the form:

type foo32 = | ONE32 | TWO32 | THREE32
val int_to_foo32 : int32 -> foo32 option
val foo32_to_int : foo32 -> int32
val foo32_to_string : foo32 -> string
val string_to_foo32 : string -> foo32 option
val compare_foo32 : foo32 -> foo32 -> int
type bar16 = | ONE | TWO | FOUR | FIVE
val int_to_bar16 : int -> bar16 option
val bar16_to_int : bar16 -> int
val bar16_to_string : bar16 -> string
val string_to_bar16 : string -> bar16 option
val compare_bar16 : bar16 -> bar16 -> int

Comparisons will be done relatively to the constructor ids.

You can also add a (sexp) decorator to output s-expression convertors for use with the sexplib library.

[%%cenum
type foo64 =
  | ONE64
  | TWO64
  | THREE64
  [@@uint64_t] [@@sexp]
]

And sexp_of_foo64 and foo64_of_sexp functions will also be available. The representation of the Sexp is the string representation of the enum.

If you do use the sexp decorator, then you will also need to add sexplib to the dependency list for your package (both in the dune file and the opam file).

Please see the ppx_test/ directory for more in-depth examples.

ocaml-cstruct's People

Contributors

avsm avatar bikallem avatar bonkf avatar c-cube avatar dinosaure avatar djs55 avatar drup avatar dsheets avatar emillon avatar fdopen avatar gasche avatar haesbaert avatar hannesm avatar jnfoster avatar jonludlam avatar let-def avatar lpw25 avatar misterda avatar mor1 avatar nojb avatar pqwy avatar pveber avatar reynir avatar rgrinberg avatar samoht avatar talex5 avatar vbmithr avatar xvilka avatar yallop avatar yomimono avatar

Stargazers

 avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar

Watchers

 avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar

ocaml-cstruct's Issues

ppx not working with lwt

Is it correct to have the file standalone.ml built into the ppx library for jbuilder (as it appears to be)?

The lwt syntax extension isn't playing nice with cstruct.

cc @diml

ocaml-cstruct-1.7.0 fails to build documentation

I wrote an ebuild (gentoo package) for a couple of packages and included support for bruild the documentation. ocaml-cstruct failed to build those. This is reproducible without the package-manager environment.

The effective command run is

        ocaml setup.ml -doc

after

        ocaml setup.ml -build

This is the output:

 * Package:    dev-ml/ocaml-cstruct-1.7.0
 * Repository: tbc
 * USE:        abi_x86_64 amd64 camlp4 doc elibc_glibc kernel_linux lwt ocamlopt userland_GNU
 * FEATURES:   preserve-libs sandbox userpriv usersandbox
>>> Unpacking source...
>>> Unpacking ocaml-cstruct-1.7.0.tar.gz to /tmp/portage/dev-ml/ocaml-cstruct-1.7.0/work
>>> Source unpacked in /tmp/portage/dev-ml/ocaml-cstruct-1.7.0/work
>>> Preparing source in /tmp/portage/dev-ml/ocaml-cstruct-1.7.0/work/ocaml-cstruct-1.7.0 ...
>>> Source prepared.
>>> Configuring source in /tmp/portage/dev-ml/ocaml-cstruct-1.7.0/work/ocaml-cstruct-1.7.0 ...

Configuration: 
ocamlfind: ........................................... /usr/bin/ocamlfind
ocamlc: .............................................. /usr/bin/ocamlc.opt
ocamlopt: ............................................ /usr/bin/ocamlopt.opt
ocamlbuild: .......................................... /usr/bin/ocamlbuild
Package name: ........................................ cstruct
Package version: ..................................... 1.7.0
os_type: ............................................. Unix
system: .............................................. linux
architecture: ........................................ amd64
ccomp_type: .......................................... cc
ocaml_version: ....................................... 4.02.3
standard_library_default: ............................ /usr/lib64/ocaml
standard_library: .................................... /usr/lib64/ocaml
standard_runtime: .................................... /usr/bin/ocamlrun
bytecomp_c_compiler: ................................. x86_64-pc-linux-gnu-gcc -march=core-avx2 -O2 -pipe -O -fno-defer-pop -Wall -D_FILE_OFFSET_BITS=64 -D_REENTRANT -O -fPIC
native_c_compiler: ................................... x86_64-pc-linux-gnu-gcc -march=core-avx2 -O2 -pipe -O  -D_FILE_OFFSET_BITS=64 -D_REENTRANT
model: ............................................... default
ext_obj: ............................................. .o
ext_asm: ............................................. .s
ext_lib: ............................................. .a
ext_dll: ............................................. .so
default_executable_name: ............................. a.out
systhread_supported: ................................. true
Install architecture-independent files dir: .......... /usr
Install architecture-dependent files in dir: ......... $prefix
User executables: .................................... $exec_prefix/bin
System admin executables: ............................ $exec_prefix/sbin
Program executables: ................................. $exec_prefix/libexec
Read-only single-machine data: ....................... $prefix/etc
Modifiable architecture-independent data: ............ $prefix/com
Modifiable single-machine data: ...................... $prefix/var
Object code libraries: ............................... /usr/lib64
Read-only arch-independent data root: ................ $prefix/share
Read-only architecture-independent data: ............. $datarootdir
Info documentation: .................................. $datarootdir/info
Locale-dependent data: ............................... $datarootdir/locale
Man documentation: ................................... $datarootdir/man
Documentation root: .................................. /usr/share/doc/ocaml-cstruct-1.7.0/html
HTML documentation: .................................. $docdir
DVI documentation: ................................... $docdir
PDF documentation: ................................... $docdir
PS documentation: .................................... $docdir
Prepend a path when installing package: .............. /tmp/portage/dev-ml/ocaml-cstruct-1.7.0/image/
findlib_version: ..................................... 1.5.5
is_native: ........................................... true
suffix_program: ...................................... 
Remove a file.: ...................................... rm -f
Remove a directory.: ................................. rm -rf
Turn ocaml debug flag on: ............................ false
Turn ocaml profile flag on: .......................... false
Compiler support generation of .cmxs.: ............... true
OCamlbuild additional flags: ......................... 
build the Lwt library: ............................... true
build the Async_unix library: ........................ false
build the UNIX library: .............................. true
build the camlp4 syntax extension: ................... true
Create documentations: ............................... true
Compile tests executable and library and run them: ... false
pkg_bigarray: ........................................ /usr/lib64/ocaml
pkg_ocplib_endian: ................................... /usr/lib64/ocaml/ocplib-endian
pkg_ocplib_endian_bigstring: ......................... /usr/lib64/ocaml/ocplib-endian
pkg_sexplib: ......................................... /usr/lib64/ocaml/sexplib
pkg_sexplib_syntax: .................................. /usr/lib64/ocaml/sexplib
pkg_lwt_unix: ........................................ /usr/lib64/ocaml/lwt
pkg_unix: ............................................ /usr/lib64/ocaml
pkg_camlp4_lib: ...................................... /usr/lib64/ocaml/camlp4
pkg_camlp4_quotations_r: ............................. /usr/lib64/ocaml/camlp4
pkg_camlp4_extend: ................................... /usr/lib64/ocaml/camlp4
ocamldoc: ............................................ /usr/bin/ocamldoc

>>> Source configured.
>>> Compiling source in /tmp/portage/dev-ml/ocaml-cstruct-1.7.0/work/ocaml-cstruct-1.7.0 ...
Finished, 1 target (0 cached) in 00:00:00.
+ /usr/bin/ocamlfind ocamlc -c -g -annot -bin-annot -principal -ccopt -O3 -ccopt -Wall -package bigarray -package ocplib-endian -package ocplib-endian.bigstring -package sexplib -syntax camlp4o -package sexplib.syntax -I lib -o lib/cstruct.cmo lib/cstruct.ml
File "lib/cstruct.ml", line 160, characters 12-25:
Warning 3: deprecated: String.create
Use Bytes.create instead.
File "lib/cstruct.ml", line 286, characters 12-25:
Warning 3: deprecated: String.create
Use Bytes.create instead.
File "lib/cstruct.ml", line 314, characters 10-23:
Warning 3: deprecated: String.create
Use Bytes.create instead.
File "lib/cstruct.ml", line 417, characters 12-25:
Warning 3: deprecated: String.create
Use Bytes.create instead.
+ /usr/bin/ocamlfind ocamlopt -c -g -annot -bin-annot -principal -ccopt -O3 -ccopt -Wall -package bigarray -package ocplib-endian -package ocplib-endian.bigstring -package sexplib -syntax camlp4o -package sexplib.syntax -I lib -o lib/cstruct.cmx lib/cstruct.ml
File "lib/cstruct.ml", line 160, characters 12-25:
Warning 3: deprecated: String.create
Use Bytes.create instead.
File "lib/cstruct.ml", line 286, characters 12-25:
Warning 3: deprecated: String.create
Use Bytes.create instead.
File "lib/cstruct.ml", line 314, characters 10-23:
Warning 3: deprecated: String.create
Use Bytes.create instead.
File "lib/cstruct.ml", line 417, characters 12-25:
Warning 3: deprecated: String.create
Use Bytes.create instead.
Finished, 33 targets (0 cached) in 00:00:03.
Finished, 0 targets (0 cached) in 00:00:00.
Solver failed:
  Ocamlbuild knows of no rules that apply to a target named Cstruct.odoc. This can happen if you ask Ocamlbuild to build a target with the wrong extension (e.g. .opt instead of .native) or if the source files live in directories that have not been specified as include directories.
Compilation unsuccessful after building 0 targets (0 cached) in 00:00:00.
E: Failure("Command ''/usr/bin/ocamlbuild' doc/api.docdir/index.html' terminated with error code 6")
 * ERROR: dev-ml/ocaml-cstruct-1.7.0::tbc failed (compile phase):
 *   (no error message)
 *
 * Call stack:
 *     ebuild.sh, line   93:  Called src_compile
 *   environment, line 2078:  Called oasis_src_compile
 *   environment, line 1858:  Called die
 * The specific snippet of code:
 *           ocaml setup.ml -doc || die;
 *
 * If you need support, post the output of `emerge --info '=dev-ml/ocaml-cstruct-1.7.0::tbc'`,
 * the complete build log and the output of `emerge -pqv '=dev-ml/ocaml-cstruct-1.7.0::tbc'`.
 * The complete build log is located at '/var/log/portage/dev-ml:ocaml-cstruct-1.7.0:20150905-104457.log'.
 * For convenience, a symlink to the build log is located at '/tmp/portage/dev-ml/ocaml-cstruct-1.7.0/temp/build.log'.
 * The ebuild environment file is located at '/tmp/portage/dev-ml/ocaml-cstruct-1.7.0/temp/environment'.
 * Working directory: '/tmp/portage/dev-ml/ocaml-cstruct-1.7.0/work/ocaml-cstruct-1.7.0'
 * S: '/tmp/portage/dev-ml/ocaml-cstruct-1.7.0/work/ocaml-cstruct-1.7.0'
 *
 * The following package has failed to build, install, or execute postinst:
 *
 *  (dev-ml/ocaml-cstruct-1.7.0:0/0::tbc, ebuild scheduled for merge), Log file:
 *   '/var/log/portage/dev-ml:ocaml-cstruct-1.7.0:20150905-104457.log'
 *

The respective ebuild (gentoo build script) can be found here.

Build assumes the existence of lwt.unix when lwt is installed.

cstruct.2.3.0 build fails in MinGW OCaml when lwt is installed. It seems that cstruct assumes the existence of lwt.unix. In MinGW OCaml, lwt does not install lwt.unix.

The workaround for cstruct.2.3.0 in MinGW OCaml + lwt is pretty simple: drop the line of <lwt/*.ml{,i,y}>: pkg_lwt.unix from the file _tag. It would be nice if the build checks lwt.unix.

It would be nice if future releases of cstruct checks lwt.unix is installed or not.

remove camlp4 dependency via ocplib-endian

ocplib-endian depends on cppo which depends on camlp4.

I think we dont need the abstraction provided by ocplib-endian any more, as our minimum compiler version can safely be 4.02.3 upwards. Creating this issue to see how invasive that patch would be...

type uint32 = int32 is risky

Cstruct declares uint32 = int32. However, int32 is signed and doing Int32.to_int may result in a negative number, which the calling code is unlikely to be expecting. Perhaps we should use a separate type for this, with safe accessor functions.

For example, ocaml-9p does:

let big_enough_for name buf needed =
  let length = Cstruct.len buf in
  if length < needed
  then error_msg "%s: buffer too small (%d < %d)" name length needed
  else return ()

let read rest =
  Fid.read rest
  >>= fun (fid, rest) ->
  Int64.read rest
  >>= fun (offset, rest) ->
  Int32.read rest
  >>= fun (len, rest) ->
  let len = Int32.to_int len in
  big_enough_for "Write.data" rest len
  >>= fun () ->
  let data = Cstruct.sub rest 0 len in
  let rest = Cstruct.shift rest len in
  return ({ fid; offset; data }, rest)

This can raise an exception (which the code is trying to avoid) for large values of len.

No bool type

Current cstruct types are all integer, 8bit upwards.
Is it possible to have a single-bit type (bool)?

I am trying to implement a protocol that has such a single-bit field.

Arithmetic overflow in set_len

set_len should

@raise Invalid_argument if [len] exceeds the size of the buffer.

However:

# Cstruct.(set_len ((sub (create 10) 5 5)) max_int );;
- : Cstruct.t = {Cstruct.buffer = <abstr>; off = 5; len = 4611686018427387903}

More dramatic illustration:

# Cstruct.(blit (create 0xfffff) 0 (set_len ((sub (create 10) 5 5)) max_int ) 0 0xfffff);;
Segmentation fault

Negative indexing and bounds-checking.

Bounds checking works on large indices, but what about the negative ones?

It's easy to "escape" a view that way, e.g.

let cs = Cstruct.create 4
let cs' = Cstruct.sub cs 2 2
let cs'' = Cstruct.sub cs' (-2) 4

Is this the intended semantics? I would assume not, as views are prohibited from growing to the right, so it makes sense to make the same guarantee on the left side.

I'm happy to provide a patch if you agree.

add 'with sexp'?

Since we're adding 'with sexp' throughout the stack, it would be helpful if Cstruct.t was defined with sexp. For reference in ocaml-xenstore I've got this:

module Cstruct = struct
  include Cstruct
  type _t = string with sexp
  let t_of_sexp s =
    let _t = _t_of_sexp s in
    let c = Cstruct.create (String.length _t) in
    Cstruct.blit_from_string _t 0 c 0 (Cstruct.len c);
    c
  let sexp_of_t t =
    let _t = Cstruct.to_string t in
    sexp_of__t _t
end

of_bytes?

What is the best way to take a Bytes.t and get it into a Cstruct.t?

The best I know of right now is Cstruct.of_string (Bytes.to_string bytes) which copies unnecessarily and is quite clumsy. Is there a better way? Should there be an of_bytes in the interface?

4.03 compatibility

Trying to build on 4.03.0+flambda using opam gives this error:

File "ppx/ppx_cstruct.ml", line 301, characters 30-40:
Error: This variant expression is expected to have type Parsetree.constant
       The constructor Const_char does not belong to type Parsetree.constant
Hint: Did you mean Pconst_char?
Command exited with code 2.

ppx_cstruct seems broken on 4.04

Two independent ppx_struct users, arp and mirage-profile, are broken on 4.04+beta on what seems to be a ppx_cstruct issue:

http://opam.ocamlpro.com/builder/html/mirage-profile/mirage-profile.0.7.0/d5a819fca03c2e5d0f175590b6022d12

+ /home/lefessan/builder/4.04.0+beta1/.opam/4.04.0+beta1/bin/ocamlfind ocamldep -package cstruct -package cstruct.ppx -package lwt -package ocplib-endian.bigstring -modules lib/trace.mli > lib/trace.mli.depends
File "lib/trace.mli", line 1:
Error: External preprocessor does not produce a valid file
Command line: ppx_cstruct '/tmp/camlppx916bcf' '/tmp/camlppx918db5'

http://opam.ocamlpro.com/builder/html/arp/arp.0.1.1/f4ae306471f10b5f44ffa3090684c677

+ ocamlfind ocamldep -package cstruct -package cstruct.ppx -package result -modules src/arp_wire.ml > src/arp_wire.ml.depends
File "src/arp_wire.ml", line 1:
Error: External preprocessor does not produce a valid file
Command line: ppx_cstruct '/tmp/camlppxeb84fe' '/tmp/camlppx986d5f'

create doesn't zero the struct

The structure returned by create isn't initialised. In a Mirage service, it might contain confidential data (e.g. TLS key material), which could easily be leaked.

Possibly this should be fixed in OCaml's Bigarray (and String.create), but at the very least, the documentation should mention this.

"noalloc" deprecation warnings when building

FWIW

File "lib/cstruct.ml", line 160, characters 0-138:
Warning 3: deprecated: [@@noalloc] should be used instead of "noalloc"
File "lib/cstruct.ml", line 162, characters 0-132:
Warning 3: deprecated: [@@noalloc] should be used instead of "noalloc"
File "lib/cstruct.ml", line 164, characters 0-132:
Warning 3: deprecated: [@@noalloc] should be used instead of "noalloc"
File "lib/cstruct.ml", line 166, characters 0-132:
Warning 3: deprecated: [@@noalloc] should be used instead of "noalloc"
File "lib/cstruct.ml", line 168, characters 0-132:
Warning 3: deprecated: [@@noalloc] should be used instead of "noalloc"
File "lib/cstruct.ml", line 170, characters 0-117:
Warning 3: deprecated: [@@noalloc] should be used instead of "noalloc"
File "lib/cstruct.ml", line 172, characters 0-102:
Warning 3: deprecated: [@@noalloc] should be used instead of "noalloc"

Provide recv_msg and send_msg

Lwt_bytes provides recv_msg and send_msg that take a list of Cstruct.t—called io_vector there. I think that these calls should also be provided by ocaml-cstruct.

Length prefixed fields and custom field types

Is there any interest in having array lengths dependent on the value of a previous field, eg length prefixed structures? For example:

cstruct string {
  uint16_t len;
  uint8_t buf[len];
} as big_endian

While technically incompatible with pure C structs, modifying ocaml-cstruct seems like the best way to avoid writing similar binary de/serializers (I've tried! [1]).

Also, how about custom field types?

cstruct packet {
  string data;
} as big_endian

The ability to go wild and combine length prefixed fields with user-defined structs would also be very helpful:

cstruct packet {
  uint8_t nb_headers;
  string headers[nb_headers];
  string data;
} as big_endian

I'm happy to do the work if there's interest in incorporating this. Also unsure if I should base this off the ppx conversion, if/when that stabilizes.

[1] Before I realized ocaml-cstruct was a thing: https://github.com/j0sh/netser

Crash in cstub for check_alignment_bigstring

When you pass a 0 argument to check_alignment the process crashes due to a division-by-zero error in the c stub.

.mli:

val check_alignment : t -> int -> bool
(** [check_alignment cstr alignment] is [true] if the first byte stored
    within [cstr] is at a memory address where [address mod alignment = 0],
    [false] otherwise.
    Typical uses are to check a buffer is aligned to a page or disk sector
    boundary. *)

cstruct.ml:

external check_alignment_bigstring : buffer -> int -> int -> bool = "caml_check_alignment_bigstring"

let check_alignment t alignment = check_alignment_bigstring t.buffer t.off alignment

cstruct_stubs.c:

CAMLprim value
caml_check_alignment_bigstring(value val_buf, value val_ofs, value val_alignment)
{
  uint64_t address = (uint64_t) (Caml_ba_data_val(val_buf) + Long_val(val_ofs));
  int alignment = Int_val(val_alignment);
  return Val_bool(address % alignment == 0);
}

When val_alignment is 0 this crashes the program.

Cstruct.t phantom type

Before cstruct-1.0, we need to make sure that Cstruct.t is a phantom type of form 'a Cstruct.t. The syntax extension will have an attach operation that ensure that the passed in Bigarray is of a sufficient length, and then all subsequent operations will be non-bounds checked (but still safe, as the attach must have happened before they will work).

missing value checks

Should cstruct check that the value of a set_XXX is in range? At the moment the following assertion does not hold (set/get_uint8 is interchangable with uint16):

let c = Cstruct.create 1 in
let v = 256 in
Cstruct.set_uint8 c 0 v ;
let v' = Cstruct.get_uint8 c 0 in
assert (v = v')

Opinions?

Missing API for a common idiom ?

I've recently needed to set N bytes of a Cstruct.t to a predefined value, literally just zero-out N bytes of a Cstruct.t, I could do it by creating a Bytes and doing blit_from_string, but it feels awkward.
It seems there is no straight-forward way to do it, and it seems something quite common:

This is the code in question:

  let pad_len = 300 - partial_len in
      let () =
        for i = 0 to pad_len do
          Cstruct.set_uint8 options_end i 0
        done
      in
      Cstruct.shift options_end pad_len

Is there a better way to accomplish this ? If not, would a patch to add such functionality be welcome ?

24 bit types

I notice a comment in ocaml-tls that should be integrated into cstruct:

(* HACK: 24 bits type not in cstruct *)
let get_uint24_len buf =
  (Cstruct.BE.get_uint16 buf 0) * 0x100 + (Cstruct.get_uint8 buf 2)

let set_uint24_len buf num =
  Cstruct.BE.set_uint16 buf 0 (num / 0x100);
  Cstruct.set_uint8 buf 2 (num mod 0x100)

Better interaction of the core type with sexplib

It is currently impossible to directly use Cstruct.t in a structure decorated with sexplib's with sexp, as this generates calls to Cstruct.sexp_of_t and Cstruct.t_of_sexp.

An optionally compiled module providing this would be neat. It would maybe make sense if this module contained a full copy of Cstruct, augmented with sexp conversions, as this would allow one to use sexps by simply opening it without modifying any of the places that mention Cstruct.t.

Additionally, it's a little unclear how to represent cstructs in sexps (hex? strings with octal escapes?).

Equality

Equality on cstructs would be really useful.

Currently, one can either Cstruct.to_bigarray both cstructs and use bigarray equality, or walk down cstructs in OCaml. Plus, it gets reimplemented around.

A vote for a fast, canonical equality.

feature request- enumerations

it would be nice to have some way to generate enumerations in the usual way, with the necessary conversion functions.

eg., something like (no idea if this is plausible):

enum Ty {
  T1,
  T2,
  Tn = 4
}

would generate equivalent of

Ty = T1 | T2 | Tn
 let ty_to_int = function
  T1 -> 0
  T2 -> 1
  Tn -> 4
let int_to_ty = function
  0 -> T1
  1 -> T2
  4 -> Tn
  _ -> invalid_arg "int_to_ty" 
let ty_to_string = function 
   ...

etc.

Incorrect version constraints specified

Ran into this trying to build a switch that relies on ocaml-graphql-server and reason. The problem comes up when installing cohttp transitively, I believe, ultimately with cstruct fails to build because # Error: No implementations provided for the following modules: Ast_404 referenced from /Users/s/.opam/ribbit2/lib/ppx_tools_versioned/ppx_tools_versioned.cmxa(Ast_convenience_404)

Here's a gist with the full output: https://gist.github.com/sgrove/f7a7a96555a804352467dff06d748ed8

I was told it's because of mis-specified version constraints, but I'm not sure which are mis-specified. Perhaps @jordwalke or @rgrinberg might have some idea?

Cannot install with opam (jbuilder-related?)

I'm on this commit currently:

commit 62321881c962d825427f10c54a1564d3055ce812
Merge: 299a30b 5cd4e6a
Author: Anil Madhavapeddy <[email protected]>
Date:   Wed May 10 23:18:48 2017 +0100

    Merge pull request #133 from avsm/jbuild
    
    port to jbuilder

When I do:

~/ocaml/ocaml-cstruct$ opam pin add cstruct file://$(pwd)

I get the following error:

=-=- Installing packages =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
Building cstruct.2.4.1:
  ./configure --prefix /DEV.PUB.OCAML/.opam/4.04.0 --enable-lwt --enable-ppx --disable-async --enable-unix
  make
  make install
  make js-install
[ERROR] The compilation of cstruct.2.4.1 failed.
Removing cstruct.2.4.1.
  make js-uninstall
  ocamlfind remove cstruct


#=== ERROR while installing cstruct.2.4.1 =====================================#
Internal error:
  "./configure": command not found.

Support 32bit?

Seems that the offset/length are currently int which limit the usability on 32bit architectures to 16MiB buffers. This is annoying, but changing offsets and lenght to int64 is a large API break that I would like to avoid.

So it might make sense to work with in64 internally, but keep the same signatures for len and offset, but expose len64 and offset64 which returns something valid (and make the normal function fail in case of overflow). Not sure if the runtime coast is worth the pain though ...

unnecessary hard type_conv dependency

type_conv pulls in camlp4, but there might be individuals who want to build cstruct without camlp4 support -- i'm not sure what the correct formulation in opam is, but using sth like "--%{camlp4+type_conv:enable}%-camlp4", plus moving type_conv to depopts should do the trick!?!

Enormous exceptions

I'm getting ever bigger Cstruct exception strings. In utop:

utop # require "cstruct";;
utop # let c = Cstruct.create 1;;
val c : Cstruct.t = {Cstruct.buffer = <abstr>; off = 0; len = 1} 
utop # let c' = Cstruct.sub c 5 6;;
Exception: Invalid_argument "Cstruct.sub: [0,1](1) off=5 len=6".
utop # let c' = Cstruct.sub c 5 6;;
Exception: Invalid_argument "Cstruct.sub: [0,1](1) off=5 len=6Cstruct.sub: [0,1](1) off=5 len=6". 
utop # let c' = Cstruct.sub c 5 6;;
Exception: Invalid_argument "Cstruct.sub: [0,1](1) off=5 len=6Cstruct.sub: [0,1](1) off=5 len=6Cstruct.sub: [0,1](1) off=5 len=6"

This is in version:

$ ocamlfind list | grep cstruct
cstruct             (version: 2.3.0)

Presumably this is a mishandling of buffers in the formatting code?

License file is missing

While the source indicates that this is MIT licensed, without the license file its hard to tell at a glance. For companies with a legal department it could also be an issue of which files are open and which are not. If you add a license file that just removes all the confusion.

Unrecognized jbuilder switch in opam package

I'm trying to install this package from opam. I get the following error:

[ERROR] The compilation of cstruct failed at "jbuilder build -p cstruct -j 4".

#=== ERROR while installing cstruct.3.0.1 =====================================#
# opam-version 1.2.2
# os           darwin
# command      jbuilder build -p cstruct -j 4
# path         /Users/milodavis/.opam/4.03.0/build/cstruct.3.0.1
# compiler     4.03.0
# exit-code    2
# env-file     /Users/milodavis/.opam/4.03.0/build/cstruct.3.0.1/cstruct-74554-1872ec.env
# stdout-file  /Users/milodavis/.opam/4.03.0/build/cstruct.3.0.1/cstruct-74554-1872ec.out
# stderr-file  /Users/milodavis/.opam/4.03.0/build/cstruct.3.0.1/cstruct-74554-1872ec.err
### stderr ###
# jbuilder: unknown option '-p'.
# jbuild TARGETS
#   -j JOBS     concurrency
#   -drules     show rules
#   -ddep-path  show depency path of errors
#   -dfindlib   debug findlib stuff
#   -help       Display this list of options
#   --help      Display this list of options



=-=- Error report -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=  🐫 
The following actions failed
  ∗  install cstruct 3.0.1
No changes have been performed

make to/of_string functions (%%cenum) optional

The definition:

[%%cenum
type foo64 =
  | ONE64
  | TWO64
  | THREE64
  [@@uint64_t]
]

currently generates

int_to_foo64 : int -> foo64 option
foo64_to_int : foo64 -> int
foo64_to_string : foo64 -> string
string_to_foo64 : string -> foo64 option

Where often only the int_to_foo64/foo64_to_int are used. Can we introduce a [@@string] to provide these conversions? Doing so reduces the binary size drastically (since there don't need to be the literal strings anymore).

There is already a [@@sexp], which emits two more functions:

foo64_of_sexp : foo64 -> Sexplib.Sexp.t
sexp_of_foo64 : Sexplib.Sexp.t -> foo64

And depends on the string functionality (thus, [@@sexp] should imply [@@string]).

host_endian appears to be broken

cstruct bpf_hdr {
    uint32_t     bh_sec;
    uint32_t     bh_usec;
    uint32_t     bh_caplen;
    uint32_t     bh_datalen;
    uint16_t     bh_hdrlen;
} as host_endian

Fails to find Cstruct.HE while little_endian and big_endian work fine:
Error: Unbound module Cstruct.HE

OS X with ocaml 4.02.3

copyv offset & length

Is there a reason why copyv does not require an offset and length?

It seems to me that it would be more sensible to have:

val to_stringv : t list -> string
val copyv : t list -> int -> int -> string

A .js subpackage?

I was just experimenting creating a 'tutorial' iocamljs notebook for [mirage/ocaml-vchan] in the style of [avsm/ocaml-dockerfile]. Several Mirage libraries use a small number of C bindings which would need to be implemented somewhere for javascript.

Does it make sense to create a cstruct.js which would contain the needed external functions?

New ppx breaks merlin?

From a clone of the latest master branch. This is using OCaml 4.04.0+flambda, merlin 2.5.4 and jbuilder 1.0+beta9.

I suspect ppx because of this:

(merlin) project flags: unknown flag: --as-ppx"

More complete error dump:

$ vim ppx_test/basic.ml
sh: -c: line 0: unexpected EOF while looking for matching `"'
sh: -c: line 1: syntax error: unexpected end of file
(merlin) backtrace:
(merlin) project flags: unknown flag: --as-ppx"
Error detected while processing function merlin#Register[117]..merlin#LoadProject:
line    2:
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/Users/hcarty/.opam/4.04.0+flambda/share/merlin/vim/autoload/merlin.py", line 740, in setup_merlin
    failures = command("project","get")
  File "/Users/hcarty/.opam/4.04.0+flambda/share/merlin/vim/autoload/merlin.py", line 201, in command
    return merlin_process().command(context(cmd))
  File "/Users/hcarty/.opam/4.04.0+flambda/share/merlin/vim/autoload/merlin.py", line 168, in command
    raise Error(value)
merlin.Error: {u'message': u'Error while running external preprocessor\nCommand line: "../_build/default/.ppx/ppx_cstruct/ppx.exe \'/var/folders/7w/pj7qnfc91jg9bn79pbsb14g80000gp/T/camlppxc6e34b\' \'/var/folders/7w/pj7qnfc91jg9bn79pbsb14g80000gp/T/camlppx60800a\' 1>/dev/null 2>/dev/null', u'valid': True, u'type': u'type', u'sub': []}sh: -c: line 0: unexpected EOF while looking for matching `"'
sh: -c: line 1: syntax error: unexpected end of file

with several more repetitions of the same or similar errors.

Safer handling of aligned buffers?

In Mirage's low-level I/O code we often require buffers to be aligned e.g.

  • page-aligned for a Xen inter-domain ring
  • sector-aligned for a Linux O_DIRECT read/write

At the moment we use the [mirage/io-page] library to allocate page-aligned bigarrays (which being page-aligned satisfy both requirements above). Other libraries accept ad-hoc mixtures of Io_page.t and Cstruct.t which leads to inefficiency [mirage/mirage-platform#110], bugs and general confusion. We tend to either

  • use a Io_page.t to spot bugs where unaligned buffers are accidentally produced; but then we end up calling functions like Io_page.to_pages which calls Bigarray.Array1.sub which is much slower than a plain Cstruct.sub; or
  • use a Cstruct.t for convenience but then we lose track of our alignment and accidentally submit an badly-aligned buffer and screw up an I/O. Sometimes we remember to check the alignment and stop with a fatal error, other times we forget and then the wrong buffer fragment is used (!)

Can we make Cstruct.t a bit more alignment-aware? Perhaps if we had a phantom type (see #14) we could make our low-level functions accept something like a [> Page_aligned ] Cstruct.t` and perhaps functions like

  • to_pages: [> Page_aligned ] Cstruct.t -> [> Page_aligned ] Cstruct.t list
  • align: 'a Cstruct.t -> [ Page_aligned ] Cstruct.t(where the returned buffer would beCstruct.sub`ed from the original to make it aligned)

Support for RT_SIGNAL_LEASE and sendfile?

"Protecting" write from a mmap'ed with RT_SIGNAL_LEASE would allow the write to terminate reporting the number of bytes written before a possible interruption by, say, another process truncating the file. Would that be useful in Unix_cstruct and/or Lwt_cstruct?

In the same vein, Lwt_cstruct.sendfile may be useful (for example to send a big file on the wire using Cohttp). A similar function seems to be available on Windows.

Your thoughts?

feature request: random, valid cenums?

I frequently find myself wanting to generate a random, but valid, cenum for testing that my code behaves correctly with potentially strange inputs. I'd like it if cstruct.ppx could help me by generating a random_cenum (just as there are int_to_cenum string_to_cenum functions generated) that chooses between the valid options.

Unit-tests are failing under windows

See https://ci.appveyor.com/project/samoht/ocaml-git-7vmk0/build/1.0.29#L484

#=== ERROR while compiling cstruct.1.9.0 ======================================#
# opam-version         1.3.0~dev (308be376d177c155c1f7f86a8ffaf621c9029836)
# os                   win32
# command              ./test.sh
# path                 C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/build/cstruct.1.9.0
# exit-code            2
# env-file             C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/build/cstruct.1.9.0\cstruct-260-2628a0.env
# stdout-file          C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/build/cstruct.1.9.0\cstruct-260-2628a0.out
# stderr-file          C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/build/cstruct.1.9.0\cstruct-260-2628a0.err
### stdout ###
# [...]
# deadbeef deadbeef
# foo = {
#   a = 0x7
#   b = 0x2c
#   c = 0xbeef
#   d = <buffer uint8_t[8] d>
#61 62 63 64 65 66 67 68 
# 
# }
# "\007\000,\000\000��abcdefgh"
# enum
# pcap
### stderr ###
# [...]
# + cp lib_test/pcap.ml _build/lib_test/pcap.ml
# + camlp4orf -printer o _build/syntax/cstruct-syntax.cma lib_test/pcap.ml
# + camlp4orf -printer o _build/syntax/cstruct-syntax.cma lib_test/pcap.mli
# + ocamlopt -pp 'camlp4orf _build/syntax/cstruct-syntax.cma' -I _build/lib -I _build/unix -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/ocaml -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/ocaml -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/sexplib -i lib_test/pcap.ml
# + cp _build/lib_test/pcap.inferred.mli _build/lib_test/pcap.mli
# + rm -f _build/lib_test/pcap.cmi
# + cd _build/lib_test
# + ocamlopt -pp 'camlp4orf ../syntax/cstruct-syntax.cma' -I ../lib -I ../unix -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/ocaml -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/ocaml -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/sexplib -c pcap.mli
# + ocamlopt -pp 'camlp4orf ../syntax/cstruct-syntax.cma' -I ../lib -I ../unix -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/ocaml -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/ocaml -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/sexplib -c pcap.ml
# + ocamlopt -I ../lib -I ../unix -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/ocplib-endian bigstring.cmxa -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/ocaml unix.cmxa -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/ocaml bigarray.cmxa -I C:/cygwin/home/appveyor/.opam/4.02.3+mingw64c/lib/sexplib sexplib.cmxa cstruct.cmxa unix_cstruct.cmxa pcap.cmx -o pcap.opt
# + ./pcap.opt
# Fatal error: exception Invalid_argument("Cstruct.iter [24,36](60) i=1 len=6619268")

fast comparison of cstructs

if we have two cstructs, which are separate parts of the memory, there should be a fast way to compare whether their contents is equal (slow way is to convert to bigarray and compare these)

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. 📊📈🎉

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google ❤️ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.