webmagic怎么做成分布式的？

構建了一個基于redis的分布式爬蟲實現，代碼如下：

public class RedisScheduler implements Scheduler{

private JedisPool pool;

private static final String QUEUE_PREFIX = "queue_";

private static final String SET_PREFIX = "set_";

public RedisScheduler(String host){

pool = new JedisPool(new JedisPoolConfig(), host);

}

@Override

public void push(Request request, Task task) {

Jedis jedis = pool.getResource();

//使用SortedSet進行url去重

if (jedis.zrank(SET_PREFIX+task.getUUID(),request.getUrl())==null){

//使用List保存隊列

jedis.rpush(QUEUE_PREFIX+task.getUUID(),request.getUrl());

jedis.zadd(SET_PREFIX+task.getUUID(),System.currentTimeMillis(),request.getUrl());

}

}

@Override

public Request poll(Task task) {

Jedis jedis = pool.getResource();

String url = jedis.lpop(QUEUE_PREFIX+task.getUUID());

return new Request(url);

}

}