ahmetb / go-linq Goto Github PK

I have structs with multiple fields and want to group by multiple fields, how can I do it?

Is it safe to copy in multiple concurrency？

package xxx

import (
	"reflect"
	"testing"
	"strings"
	"fmt"
	linq "github.com/ahmetb/go-linq"
)

type Row map[string]interface{}

func maxMatricData(data []Row, groupKeys []string, metricKeys []string) []Row {
	if len(metricKeys) == 0 {
		return data
	}

	var maxData []Row
	q := linq.
		From(data).
		GroupBy(
			func(d interface{}) interface{} {
				r := d.(Row)
				var vals []string
				for _, k := range groupKeys {
					vals = append(vals, fmt.Sprintf("%v", r[k]))
				}
				return strings.Join(vals, ",")
			},
			func(d interface{}) interface{} { return d },
		).
		Select(func(g interface{}) interface{} {
			query := linq.From(g.(linq.Group).Group).
				OrderByDescending(func(r interface{}) interface{} { return r.(Row)[metricKeys[0]] })
			if len(metricKeys) > 1 {
				for _, k := range metricKeys[1:] {
					query = query.ThenByDescending(func(r interface{}) interface{} { return r.(Row)[k] })
				}
			}
			return query.First()
		}).
		OrderByDescending(func(r interface{}) interface{} {
			return r.(Row)[groupKeys[0]]
		})

       // FIXME: bug for more than 2rd order by Descending
	for _, k := range groupKeys[1:] {
		q = q.ThenByDescending(func(r interface{}) interface{} {
			return r.(Row)[k]
		})
	}
	q.ToSlice(&maxData)

	return maxData
}

func Test_maxMatricData(t *testing.T) {
	type args struct {
		data       []Row
		groupKeys  []string
		metricKeys []string
	}
	tests := []struct {
		name string
		args args
		want []Row
	}{
		{
			"default",
			args{
				[]Row{
					Row{"a": "a1", "b": "b3", "c": "1", "v1": 17, "v2": 21, "v3": 31},
					Row{"a": "a1", "b": "b1", "c": "1", "v1": 11, "v2": 21, "v3": 31},
					Row{"a": "a1", "b": "b2", "c": "1", "v1": 15, "v2": 21, "v3": 31},
					Row{"a": "a1", "b": "b1", "c": "1", "v1": 13, "v2": 21, "v3": 31},
					Row{"a": "a1", "b": "b2", "c": "1", "v1": 14, "v2": 21, "v3": 31},
					Row{"a": "a1", "b": "b1", "c": "1", "v1": 11, "v2": 22, "v3": 31},
					Row{"a": "a1", "b": "b2", "c": "1", "v1": 15, "v2": 21, "v3": 32},
				},
				[]string{"a", "b", "c"}, // passed when: []string{"a", "b"}
				[]string{"v1"},
			},
			[]Row{
				Row{"a": "a1", "b": "b3", "c": "1", "v1": 17, "v2": 21, "v3": 31},
				Row{"a": "a1", "b": "b2", "c": "1", "v1": 15, "v2": 21, "v3": 31},
				Row{"a": "a1", "b": "b1", "c": "1", "v1": 13, "v2": 21, "v3": 31},
			},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := maxMatricData(tt.args.data, tt.args.groupKeys, tt.args.metricKeys); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("maxMatricData() = %v, want %v", got, tt.want)
			}
		})
	}
}

testing output:

--- FAIL: Test_maxMatricData (0.00s)
    --- FAIL: Test_maxMatricData/default (0.00s)
        e:\Apps\goApps\git.code.oa.com\WeishiQA\QualityHttpServer\queries\druid\handler_test.go:44: maxMatricData() = [map[a:a1 b:b3 c:1 v1:17 v2:21 v3:31] map[a:a1 b:b1 c:1 v1:13 v2:21 v3:31] map[a:a1 b:b2 c:1 v1:15 v2:21 v3:31]], want [map[a:a1 b:b3 c:1 v1:17 v2:21 v3:31]  map[a:a1 b:b2 c:1 v1:15 v2:21 v3:31]  map[a:a1 b:b1 c:1 v1:13 v2:21 v3:31]]
FAIL
FAIL	WeishiQA/QualityHttpServer/queries/druid	0.277s
Error: Tests failed.

plinq.go:line 272 change
for v := range q.values {
to
for _, v := range q.values {

Codes:

func main() {
	l := []string{"foo", "bar", "baz"}
	linq.From(l).ToSlice(&l)
	fmt.Println(l)
}

Result:

[foo bar baz foo bar baz]

go-linq version:

- name: github.com/ahmetalpbalkan/go-linq
  version: 8985ec99e11a8bff7eb19dd0a0b2187770dab23a

I don't know whether this is a bug or a feature, but if this is a feature, I think it's better to have this behavior documented on ToSlice's API doc.

In the readme.md, it says

To find names of first 5 students over 18:

But the code doesn't seem to do any ordering, so it doesn't find the "first" 5 students, just any 5 students over 18.

@kalaninja https://github.com/ahmetalpbalkan/go-linq/wiki

what do you think?

Really a huge fan of LINQ. I use it all the time in the front-end (map, reduce are basically the same ideas). Would love to use this package but the one question I have is---"Is this used in production by anyone?"

concat.go中的Append方法，第12行i, ok := next()前面可判断一下appended是true则直接返回

Hi guys,

I'm trying to use the package and either I don't understand something or First returns a copy of the element. Here is a simple example:

collection := Items{
	Item{
		ID:    1,
		Title: "title 1",
	},
	Item{
		ID:    2,
		Title: "title 2",
	},
}

item1 := From(collection).FirstWith(func(m interface{}) bool {
	return m.(Item).ID == 2
})
fmt.Printf("%p \n", &item1)
item2 := From(collection).FirstWith(func(m interface{}) bool {
	return m.(Item).ID == 2
})
fmt.Printf("%p \n", &item2)

Output:

0xc4204445d0 0xc4204445e0

As you can see the same element gets two different address allocation.
How can I get a pointer to my initial element?

Instead of taking an array / a slice as input, From() should also take a channel as input. It would provide far more flexibility.

For example,

import . "github.com/ahmetalpbalkan/go-linq"

type Student struct {
    id, age int
    name string
}

func example_usage() {
    ch := make(<-chan Student)
    DBStudentToChan(ch) // some function that outputs to the channel

    over18Names, err := From(ch)
        .Where(func (s T) (bool,error){
            return s.(*Student).age >= 18, nil
        })
        .Select(func (s T) (T,error){
            return s.(*Student).name, nil
        }).Results()
}

Hello. I stil getting into go and learning. I used to do OOP languages, so excuse the newbie questions.
This is really nice to see in Golang.

Does this generate at design time ?

Have you thought about design time using clipperhouse gen ?

Have you thought about an JSON like definition of the data types, and gen the types and then gen the query layer ?

This has just randomly showed up in API 1.14 docs. I am not even sure why they modify docs after releasing it. It surely wasn't there before... https://docs.docker.com/reference/api/docker_remote_api_v1.14/#create-a-container

I am planning to change my GitHub username to ahmetb soon. If you already vendored this repo (which you should) this change is not going to break your builds, it may cause issues when you try to update the vendored package.

Leaving this issue open for a while to discuss issues that come up because of the change.

@cleitonmarx are we actually using this benchmark_test.go file in v3-dev branch?

It looks like our travis-CI tests are not executing it. I just got this output:

$ go test -bench . .
BenchmarkSelectWhereFirst-4            	 5000000	       323 ns/op
BenchmarkSelectWhereFirst_generics-4   	 1000000	      1959 ns/op
BenchmarkSum-4                         	      30	  46626334 ns/op
BenchmarkSum_generics-4                	       3	 468433125 ns/op
BenchmarkZipSkipTake-4                 	 5000000	       301 ns/op
BenchmarkZipSkipTake_generics-4        	 1000000	      1557 ns/op
PASS
ok  	github.com/ahmetalpbalkan/go-linq	12.279s

Is there a reason for this to stay in the repo? We can perhaps make it a bit more detailed (such as specify input sizes) and still keep it in the package?

Codes:

func main() {
	linq.From([]string{}).OrderBy(func(in interface{}) interface{} {
		return 0
	}).Results()
}

OrderBy on a non-empty slice which have a Where filter, and the filter will filter all the elements, will cause the bug too.

Stack trace:

panic: runtime error: index out of range

goroutine 1 [running]:
panic(0x437ce0, 0xc42000c0f0)
	/usr/local/Cellar/go/1.7.3/libexec/src/runtime/panic.go:500 +0x1a1
github.com/leancloud/lean-cli/lean/vendor/github.com/ahmetalpbalkan/go-linq.Query.sort(0xc420385170, 0xc42038ede0, 0x1, 0x1, 0xc42038edc0, 0x0, 0x743200)
	/Users/asaka/Codes/go/src/github.com/leancloud/lean-cli/lean/vendor/github.com/ahmetalpbalkan/go-linq/orderby.go:177 +0x235
github.com/leancloud/lean-cli/lean/vendor/github.com/ahmetalpbalkan/go-linq.Query.OrderBy.func1(0x45e5c0)
	/Users/asaka/Codes/go/src/github.com/leancloud/lean-cli/lean/vendor/github.com/ahmetalpbalkan/go-linq/orderby.go:28 +0x8d
github.com/leancloud/lean-cli/lean/vendor/github.com/ahmetalpbalkan/go-linq.Query.Results(0xc42038edc0, 0x546ec0, 0xc42038edc0, 0xc420385170)
	/Users/asaka/Codes/go/src/github.com/leancloud/lean-cli/lean/vendor/github.com/ahmetalpbalkan/go-linq/result.go:208 +0x2e
main.main()
	/Users/asaka/Codes/go/src/github.com/leancloud/lean-cli/lean/main.go:299 +0xf0

go-linq version:

- name: github.com/ahmetalpbalkan/go-linq
  version: 8985ec99e11a8bff7eb19dd0a0b2187770dab23a

(current master branch codes).

this sounds like a good match with boltdb, in terms of doing queries in memory.

i currently use boltdb and also riak. i kind of hate riak but am stuck with it still.
i was wondering if you think that linq would help me for writing many queries i need to do on top of boltdb ?

PLINQ removed for now (see channels support)
Where can I see channels support? Is there any example about this?
Thanks

How to implement complex SQL functions similar to the following

type TransactionRecord struct {
	ID        uint64
	AccountID string
	TCode     string
	TDate     string
	TType     uint8
	Amount    float64
}

var tds []TransactionRecord
t1 := TransactionRecord{ID: 1, AccountID: "A001", TCode: "161700", TDate: "2020-08-01", TType: 17, Amount: 23.0}
t2 := TransactionRecord{ID: 2, AccountID: "A001", TCode: "161700", TDate: "2020-08-01", TType: 17, Amount: 99.0}
t3 := TransactionRecord{ID: 3, AccountID: "A001", TCode: "161700", TDate: "2020-08-01", TType: 17, Amount: 25.0}

t4 := TransactionRecord{ID: 1, AccountID: "A001", TCode: "161700", TDate: "2020-08-01", TType: 24, Amount: 13.0}
t5 := TransactionRecord{ID: 2, AccountID: "A001", TCode: "161700", TDate: "2020-08-01", TType: 24, Amount: 69.0}
t6 := TransactionRecord{ID: 3, AccountID: "A001", TCode: "161700", TDate: "2020-08-01", TType: 24, Amount: 85.0}
		
t7 := TransactionRecord{ID: 4, AccountID: "A001", TCode: "161700", TDate: "2020-08-02", TType: 17, Amount: 11.0}
t8 := TransactionRecord{ID: 5, AccountID: "A001", TCode: "161700", TDate: "2020-08-02", TType: 17, Amount: 33.0}
t9 := TransactionRecord{ID: 6, AccountID: "A001", TCode: "161700", TDate: "2020-08-02", TType: 17, Amount: 57.0}
		
tds = append(tds, t1)
tds = append(tds, t2)
tds = append(tds, t3)
tds = append(tds, t4)
tds = append(tds, t5)
tds = append(tds, t6)
tds = append(tds, t7)
tds = append(tds, t8)
tds = append(tds, t9)

I want to implement the following SQL functions, how to write code with go-linq package

select AccountID,TCode, TDate,TType, sum(Amount)  as am from TransactionDetails where TType = 17 or TType = 24 GROUP BY AccountID, TCode,TDate, TType ORDER BY AccountID, TCode,TDate, TType;

I have seen there are lot of methods (ex: Where. Select etc ) that return error type alongside bool type. In all your examples, you are returning nil error. So when to send error other than nil and what happens if there is an error other than nil ? Thanks

@cleitonmarx what do you think about updating the site. You can add some info/examples of the new -T methods api and add some info about yourself.

Hi, I want to union two query with a selector define the key of item, but file union.go doesn't have func UnionBy. @ahmetb

How do you orderBy alphabetically? All the examples assume int values.

First of all, thanks for simple and super useful library!!

Only thing I would like to see added is a way to add custom functions like Average. I know there are multiple functions readily available, but what if I want to do Mean, Median or some kind of custom processing on values. It doesn't expose values to outside world as well. So kind of feels restricted.

If you have any thoughts on this, please share, I can try implementing it.

reflect.Append() is more robust I think.

I'm having an issue getting a release with ToChannelT in it following the install directions for this package. Is there a planned release coming up with that or is there an alternative way to get get the package such that I get the version with that method? go get doesn't seem to be able to get by tag or at least it is not working for me.

I would like to group by the error message and get the count of it. How can I do it using linq??

example : Slice has 100 error message, actually they are repetitive.. so I was able to group them using linq and it got reduced the group of 11 type. but I want to know the count of each group. line error 1 repeated 20 times out of 100.

could a question and (hopefully an answer) be added to the FAQ along the lines of: "Are linq result slices guaranteed to be assigned new underlying arrays to avoid any memory management issues?"

For maintainers, as FYI:

https://medium.com/wesovilabs/koazee-vs-go-funk-vs-go-linq-caf8ef18584e

I know this may not related to you go-linq code.

I want to write some linq library using c, and reference your go-linq code, but it just core dumps.

Below is the c code:

#include <stdio.h>
#include <stddef.h>
#include "vec.h"

#define bool int
#define true 1
#define false 0

#define lambda(ret, body) ({ ret __fn__ body __fn__; })

typedef bool (*Iterator)(void*);
typedef Iterator (*Iterate)(void);

typedef bool (*Predicate)(void *);

typedef struct Query {
    Iterate iterate;
}Query;


Query From(void *source) {
    vec_void_t *v = (vec_void_t *)source;
    int len = v->length;

    Iterator iter() {
        int index = 0;

        bool iterator(void *item) {
            item = NULL;
            bool ok = index < len;
            if (ok) {
                item = v->data[index];
                index++;
            }
            return ok;
        }

        return iterator;
    }

    Query result;
    result.iterate = iter;
    return result;
}

Query Where(Query *q, Predicate predicateFn) {
    Iterator iter() {
        Iterator next = q->iterate();
        bool iterator(void *item) {
            bool ok = false;
            for (ok = next(item); ok; ok = next(item)) {
                if (predicateFn(item)) {
                    return ok;
                }
            }
            return ok;
        }

        return iterator;
    }

    Query result;
    result.iterate = iter;
    return result;
}

vec_void_t Results(Query *q) {
    vec_void_t v;
    vec_init(&v);
    Iterator next = q->iterate();
    bool ok = false;
    void *item = NULL;
    for (ok = next(item); ok; ok = next(item)) {
        vec_push(&v, item);
    }

    return v;
}

int main() {
    vec_void_t v;
    vec_init(&v);

    int i, j, m, n;

    i = 1;
    vec_push(&v, &i);

    j = 2;
    vec_push(&v, &j);

    m = 3;
    vec_push(&v, &m);

    n = 4;
    vec_push(&v, &n);

    int w = 0;
    for (w = 0; w < 4; w++) {
        printf("v[%d] = %d\n", w, *((int *)v.data[w]));
    }

    Query qFrom = From(&v);
    Query qWhere = Where(&qFrom, lambda(bool,  (void *v) {
                      int i = *(int *)v;
                      return i / 2 == 0;
                }));

    vec_void_t vResults = Results(&qWhere);

    for (w = 0; w < 4; w++) {
        printf("vResults[%d] = %d\n", w, *((int *)vResults.data[w]));
    }

    vec_deinit(&v);
    return 0;
}

The above c code core dump at Iterator next = q->iterate(); line of function Results.

Could you kindly tell me why the c code doesn't work? and how to fix that?

Thanks in advance.

Hi there,

first of all thanks for this cool library.
I'm wondering whether there are any plans to extend the lib in order to be able to build Expression Tree and use it for doing DB queries. Not sure, it's possible with Go's capabilities, though.

https://github.com/jochasinga/RxGo provides Reactive Extensions for Go.

I wonder how it can be combined with go-linq and if one can augment the other. Leaving this issue around here to see if anyone can come up with something creative.

I Want to implement GroupBY for array of slices, and need to apply groupBy operation on That, Can you provide GroupBy() method for the same?

I am trying to do create go.mod in my project. And this error arrives what should I do. I have imported
gopkg.in/ahmetb/go-linq.v3 this package in my project files

Go now has a prototype of generics implementation. Here are some resources:

• https://blog.golang.org/generics-next-step
• https://go.googlesource.com/proposal/+/refs/heads/master/design/go2draft-type-parameters.md
• https://rakyll.org/generics-proposal/

This potentially could give a severe performance boost to go-linq, as well as actually make this library useful.

I haven't taken a closer look at how we would do this yet. For example, we might still end up having to do some type assertions. However, it might help us redesign the package and release a v4 if the generics proposal is adopted.

I believe WhereIndexed doesn't work as design:

https://play.golang.org/p/HD5-rxIkpx0

package main

import (
	"fmt"

	"github.com/ahmetb/go-linq/v3"
)

func main() {

	r := linq.Range(1, 10).
		WhereIndexed(func(i int, _ interface{}) bool {
			return i%3 == 0
		}).
		Results()
	fmt.Println(r)
}

output of the code above is
[1 2 3 4 5 6 7 8 9 10]
instead of
[3 6 9]

maybe the bug is here

https://github.com/ahmetb/go-linq/blob/master/where.go#L61

thanks!

Currently Results() returns a slice of []T where perhaps a way to get the "original" type of the elements would be more convenient. I constantly find myself having to iterate over the []T to copy all elements to a new slice.

x, err := linq.From(f.Films).Where(func(x linq.T) (bool, error) { return true, nil }).Results()
if err != nil {
 fmt.Println(err)
}

Here x is of type []T and so I cannot reassign it to f.Films. I need to do the following:

f.Films = make([]Film, len(x))
for i, b := range x {
 f.Films[i] = b.(Film)
}

I find myself doing this kind of thing very often when using your package. Perhaps there should be a way to do this included in the package itself?

distinctFirstNames, err := From(people).DistinctBy(func (p T) (bool, error){
    return p.(*Person).FirstName
}).Results()

maybe:

distinctFirstNames, err := From(people).DistinctBy(func (p T,p2 T) (bool, error){
    return p.(*Person).FirstName==p2.(*Person).FirstName,nil
}).Results()

Hi there,

First of all thanks this cool library.

I'm thinking about there is any plans to extend this lib with func Query.Difference besides Query.Intersect and Query.Union.

I have a slice like this following:

type Post struct {
  title          string
  category  string
}

post1 := Post{"a1", "hello"}
post2 := Post{"a2", "hello"}
post3 := Post{"a3", "world"}

posts := []*Post{&post1, &post2, &post3}

I wanna extract this posts slice to a map like this:

postMap := make(map[string][]string)

So the category is string key and title is a slice value, when I call postMap["hello"] it return a slice include post1 and post2 and when I call postMap["world"] it will return post3.

How can I get this use go-linq? please help

@kalaninja can you send a small PR to gh-pages branch to update https://ahmetalpbalkan.github.io/go-linq/

we probably just need to

• update examples
• about section: either just remove my name or add both of our names (+pictures). I'm more fan of removing this section altogether.

This is an amazing library and thanks to you for this.

Can we use a semver instead of current vx.y versioning, for better third party packaging usage?

Now I have a project using glide to manage third party deps. And glide will recognize semver if a repo have tags like v2.0.0 like this, and I can keep using a specified major version of package per build, and it wouldn't break my codes.

I think it's much cooler if go-linq use semver 😎

Since 1.11 Go Modules is probably the preferred way to handle versioning.
We should add support for it!

When looking at a language such as Ruby, one can pass in an array of fields to do multi-level sorting ( i.e. "sort by last name, first name" ).

Currently it does not seem possible to do this in go-linq

@ahmetalpbalkan, I think the ForEach function can be an awesome feature for the library. I personally have some interesting use cases in my work that can be better expressed using the ForEach approach.

This code is another try to implement the @kalaninja idea:

func (q Query) ForEach(c chan interface{}, action func(interface{})) {
	next := q.Iterate()
	if c == nil {
		for item, ok := next(); ok; item, ok = next() {
			action(item)
		}
	} else {
		var wg sync.WaitGroup
		for item, ok := next(); ok; item, ok = next() {
			wg.Add(1)
			go func(wg *sync.WaitGroup, item interface{}) {
				defer wg.Done()
				action(item)
			}(&wg, item)
		}

		go func() {
			wg.Wait()
			close(c)
		}()
	}
}

This provides an elegant way to solve the concurrent example problem:

results := make(chan interface{})
domains := []string{"www.google.com", "www.github.com", "www.facebook.com"}

From(domains).
    Select(func(d interface{}) interface{} {
        return fmt.Sprintf("http://%s/favicon.ico", d)
    }).
    ForEach(results, func(url interface{}) {
	  resp, _ := http.Get(url.(string))
	  results <- resp.StatusCode
    })

count := FromChannel(results).CountWith(func(r interface{}) bool {
    return r.(int) == 200
})

fmt.Println("Count:", count)

Or just for run simple tasks in parallel:

From(domains).
    Select(func(d interface{}) interface{} {
	return fmt.Sprintf("http://%s/favicon.ico", d)
    }).
    ForEach(results, func(url interface{}) {
	resp, _ := http.Get(url.(string))
	fmt.Println(url.(string), "->", resp.StatusCode)
    })

<-results

fmt.Println("Finished all tasks")

If no channel was provided, ForEach will iterate sequentially as proposed by @kalaninja.

Hi,

This lib is awesome, but I didn't find any example for GroupBy for multiple columns, is it possible and supported?

thanks

Hi!

First of all, thanks for go-linq, it's awesome :)

Since there are no generics in Golang, we have to deal with interface{}-s.

I just spotted nice hack in Glow project to allow users to specify types on parameters themself:
https://github.com/chrislusf/glow/blob/master/flow/dataset_map.go#L19-L47

Seems like a really great idea to steal :)