Giter Club home page Giter Club logo

delayqueue's Introduction

delayqueue

Delay queues that depend on Kafka

Example

package main

import (
	"context"
	"fmt"
	"github.com/QbCheng/delayqueue/consummer"
	"github.com/QbCheng/delayqueue/producer"
	"github.com/Shopify/sarama"
	"log"
	"sync"
	"time"
)

// testDelayQueue Latency to be concerned about
var testDelayQueue = []time.Duration{
	15 * time.Second,
	30 * time.Second,
	60 * time.Second,
	10 * time.Minute,
	20 * time.Minute,
	30 * time.Minute,
	time.Hour,
	3 * time.Hour,
	6 * time.Hour,
}

var testNet = []string{"192.168.1.133:9092"}

func main() {
	dpConsumer, err := consummer.NewController("testDp", testNet, testDelayQueue)
	if err != nil {
		panic(err)
	}
	err = dpConsumer.Run(context.Background())
	if err != nil {
		panic(err)
	}

	dpProducer, err := producer.NewDQProducer("testDp", testDelayQueue, testNet)
	if err != nil {
		panic(err)
	}

	wg := &sync.WaitGroup{}
	wg.Add(100)
	for i := 0; i < 100; i++ {
		go func() {
			defer wg.Done()
			for j := 0; j < 100; j++ {
				for _, delayTime := range testDelayQueue {
					_ = dpProducer.Send(delayTime, []byte("hell World"), "testTopic")
				}
			}
		}()
	}

	handle := RealTopicHandle{}
	wg.Add(1)
	ctx, cancel := context.WithCancel(context.Background())
	go handle.Run(ctx, []string{"testTopic"}, wg)

	time.Sleep(24 * time.Hour)
	dpConsumer.Close()
	cancel()
	_ = dpProducer.Close()
}

type RealTopicHandle struct {
}

func (h RealTopicHandle) Setup(session sarama.ConsumerGroupSession) error {
	return nil
}

func (h RealTopicHandle) Cleanup(_ sarama.ConsumerGroupSession) error {
	return nil
}

func (h RealTopicHandle) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
	for msg := range claim.Messages() {
		log.Printf("Message topic:%q partition:%d offset:%d\n", msg.Topic, msg.Partition, msg.Offset)
		session.MarkMessage(msg, "")
	}
	return nil
}

func (h *RealTopicHandle) Run(ctx context.Context, topics []string, wg *sync.WaitGroup) {
	defer wg.Done()
	log.SetFlags(log.Lshortfile | log.Ltime | log.Ldate)
	config := sarama.NewConfig()
	config.Consumer.Return.Errors = true
	config.Consumer.Offsets.Initial = sarama.OffsetOldest
	group, err := sarama.NewConsumerGroup(testNet, "RealTopic-group", config)
	if err != nil {
		panic(err)
	}
	defer func() { _ = group.Close() }()

	// Track errors
	go func() {
		for err = range group.Errors() {
			log.Print("err : ", err.Error())
		}
	}()

	for {
		err = group.Consume(ctx, topics, h)
		if err == nil {
			return
		} else {
			fmt.Println(err)
		}
	}
}

delayqueue's People

Contributors

qbcheng avatar

Stargazers

 avatar xiaoye avatar

Watchers

 avatar

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    ๐Ÿ–– Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. ๐Ÿ“Š๐Ÿ“ˆ๐ŸŽ‰

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google โค๏ธ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.