Announcement Announcement Module
Collapse
No announcement yet.
[NEO4J] Very poor performances Page Title Module
Move Remove Collapse
X
Conversation Detail Module
Collapse
  • Filter
  • Time
  • Show
Clear All
new posts

  • [NEO4J] Very poor performances

    Hi all
    I'm using this environment:
    spring 3.1.1
    spring data neo4j 2.1.0.BUILD-SNAPSHOT
    spring data neo4j rest 2.2.0.BUILD-SNAPSHOT
    neo4j 1.8 in server configuration (not embedded)
    neo4j rest graphdb 1.8

    I have this Node Entity:
    User.java
    Code:
    package org.springframework.data.neo4j.examples.hellograph;
    import java.util.HashSet;
    import java.util.Set;
    import org.neo4j.graphdb.Direction;
    import org.springframework.data.neo4j.annotation.Fetch;
    import org.springframework.data.neo4j.annotation.GraphId;
    import org.springframework.data.neo4j.annotation.Indexed;
    import org.springframework.data.neo4j.annotation.NodeEntity;
    import org.springframework.data.neo4j.annotation.RelatedToVia;
    @NodeEntity
    public class User implements Comparable<User> {
    	private final static String INFLUENCE = "INFLUENCE";
    	@GraphId
    	private Long id; 
    	@Indexed(unique = true)
    	private String type; 
    	@Indexed
    	private Long index; 
    	@Indexed
    	private String dbType; 
    	@Fetch
    	@RelatedToVia(type = INFLUENCE, direction = Direction.BOTH)
    	private Set<Influence> usersList = new HashSet<Influence>();
    	private double influence;
    	public User() {}
    	public User(long index, String dbType) {
    		this.type = index + "_" + dbType;
    		this.index = index;
    		this.dbType = dbType;
    	}
    	public Long getId() {
    		return id;
    	}
    	public void setId(Long id) {
    		this.id = id;
    	}
    	public Influence addRelationship(User user, double influenceValue, int index) {
    		Influence influence = new Influence(this, user, influenceValue, index);
    		this.usersList.add(influence);
    		return influence;
    	}
    	public Long getIndex() {
    		return index;
    	}
    	public void setIndex(Long index) {
    		this.index = index;
    	}
    	@Override
    	public int hashCode() {
    		return (id == null) ? 0 : id.hashCode();
    	}
    	@Override
    	public boolean equals(Object obj) {
    		if (this == obj)
    			return true;
    		if (obj == null)
    			return false;
    		if (getClass() != obj.getClass())
    			return false;
    		User other = (User) obj;
    		if (id == null)
    			return other.id == null;
    		return id.equals(other.id);
    	}
    	@Override
    	public String toString() {
    		return String.format("User{index=%d, type='%s', dbType='%s'}", index, type, dbType);
    	}
    	public Set<Influence> getUserList() {
    		return usersList;
    	}
    	public String getType() {
    		return type;
    	}
    	public void setType(String type) {
    		this.type = type;
    	}
    	public double getInfluence() {
    		return influence;
    	}
    	public void setInfluence(double influence) {
    		this.influence = influence;
    	}
    	@Override
    	public int compareTo(User u) {
    		double val = this.getInfluence() - u.getInfluence();
    		if (val > 0) {
    			return 1;
    		}
    		if (val == 0) {
    			return 0;
    		}
    		return -1;
    	}
    	public String getDbType() {
    		return dbType;
    	}
    	public void setDbType(String dbType) {
    		this.dbType = dbType;
    	}
    }
    I have this UserRepository
    UserRepository.java
    Code:
    package org.springframework.data.neo4j.examples.hellograph;
    import java.util.Set;
    import org.springframework.data.neo4j.annotation.Query;
    import org.springframework.data.neo4j.repository.GraphRepository;
    public interface UserRepository extends GraphRepository<User> {
    	/**
    	 * 
    	 * @param type
    	 * @return il numero di tutti gli Utenti marcati con la proprietÓ type
    	 */
    	@Query("START n=node(*) WHERE n.type? =~ {0} RETURN count(n)-1")
    	public long getNumbersOfAllUsersByType(String type);
    	/**
    	 * @param type
    	 *            : il tipo di nodo da considerare
    	 * @return TUTTE le relazioni dei nodi con type = "type"
    	 */
    	@Query("START n=node(*) match (n)-[x]->(rel) where n.type? =~ {0} return rel")
    	public Set<Influence> findAllRelationshipByDbType(String type);
    	@Query("START n=node(*) match (n)-[x]->(rel) where n.type? =~ {0} return count(rel)")
    	public long getNumbersOfAllRelationshipByDbType(String type);
    }
    This is my user service
    UserService.java
    Code:
    package org.springframework.data.neo4j.examples.hellograph;
    import java.util.Set;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.data.neo4j.conversion.EndResult;
    import org.springframework.stereotype.Service;
    @Service
    public class UserService {
    	@Autowired
    	private UserRepository userRepository;
    	public User createUser(long index, String dbType) {
    		return userRepository.save(new User(index, dbType));
    	}
    	public void deleteUsers() {
    		userRepository.deleteAll();
    	}
    	public Iterable<User> createAllUser(Iterable<User> userList) {
    		return userRepository.save(userList);
    	}
    	public void save(User user) {
    		userRepository.save(user);
    	}
    	public EndResult<User> findAllUserByDbType(String type) {
    		return userRepository.findAllByPropertyValue("dbType", type);
    	}
    	public long getNumbersOfAllUsersByType(String type) {
    		String regExpr = "(?i).*" + type;
    		return userRepository.getNumbersOfAllUsersByType(regExpr);
    	}
    	public long getNumbersOfAllRelationshipByType(String DbType) {
    		String regExpr = "(?i).*" + DbType;
    		return userRepository.getNumbersOfAllRelationshipByDbType(regExpr);
    	}
    	public User findUserById(Long id) {
    		return userRepository.findOne(id);
    	}
    	public User findUserByType(String type) {
    		return userRepository.findByPropertyValue("type", type);
    	}
    	public Set<Influence> getAllRelationship(User user) {
    		return user.getUserList();
    	}
    	public void deleteAll() {
    		userRepository.deleteAll();
    	}
    	public Iterable<User> getAllUsers() {
    		return userRepository.findAll();
    	}
    	public long getNumberOfUsers() {
    		return userRepository.count();
    	}
    	public Set<Influence> findAllRelationshipByType(String DbType) {
    		String regExpr = "(?i).*" + DbType;
    		return userRepository.findAllRelationshipByDbType(regExpr);
    	}
    }
    This is my application context file:
    Code:
    <?xml version="1.0" encoding="UTF-8" standalone="no"?>
    <beans xmlns="http://www.springframework.org/schema/beans"
    	xmlns:context="http://www.springframework.org/schema/context"
    	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:neo4j="http://www.springframework.org/schema/data/neo4j"
    	xmlns:tx="http://www.springframework.org/schema/tx"
    	xsi:schemaLocation="
           	http://www.springframework.org/schema/beans 
           	http://www.springframework.org/schema/beans/spring-beans.xsd
    		http://www.springframework.org/schema/context 
    		http://www.springframework.org/schema/context/spring-context.xsd
    		http://www.springframework.org/schema/data/neo4j
    		http://www.springframework.org/schema/data/neo4j/spring-neo4j.xsd
    		http://www.springframework.org/schema/tx
    		http://www.springframework.org/schema/tx/spring-tx.xsd">
    	<context:spring-configured />
    	<context:annotation-config />
    	<context:component-scan	base-package="it.test.ne04j.sample" />
    	<neo4j:repositories base-package="it.test.ne04j.sample.repositories" />
    	<neo4j:config graphDatabaseService="graphDatabaseService" />
    	<bean id="graphDatabaseService" class="org.springframework.data.neo4j.rest.SpringRestGraphDatabase">
    		<constructor-arg index="0" value="http://localhost:7474/db/data" />
    	</bean>
    	<tx:annotation-driven mode="aspectj" />
    </beans>
    Now I was trying to create 5000 nodes and then add relations between these nodes; each node represents an User istance by this code:
    Code:
    List<User> userList = new ArrayList<User>();
      for (int i = 0; i < M; ++i) {
      User u = new User(i, type);
      userList.add(u);
    }
    StopWatch sw = new StopWatch();
    sw.start("Creazione Utenti");
    userService.createAllUser(userList);
    sw.stop();
    logger.debug("Processo " + sw.getLastTaskName()+ " terminato in :" + sw.getTotalTimeSeconds());
    Now on Windows 7 64 bit machine after 900 nodes I had this error "com.sun.jersey.api.client.ClientHandlerExcept ion: java.net.SocketTimeoutException: Read timed out"
    I switched on Ubuntu 12.04 machine 32 bit and the same code successfully completed in more or less 20 minutes.
    Then I tried to add some relations between nodes always on the ubuntu machine. Well adding these relations took more or less 10 hours
    These performances seem to me really really poor....Am I wrong in anything? How can I Improve performances?
    Thank you
    Angelo

  • #2
    Angelo,

    Using the Spring Data Neo4J REST plugin for inserting large number of nodes/relationships will not give great performance.

    see http://static.springsource.org/sprin.../html/#d5e2009

    An alternative that I am using instead of inserting over REST would be to package your UserService into a Neo4j Server Plugin. You can then create an unmanaged server extension that exposes a RESTful entry point to your UserService and get your application to call this web service to insert users.

    see http://static.springsource.org/sprin.../html/#d5e1982

    Comment


    • #3
      jarebu,

      +1, the REST interface is super chatty (as REST interfaces would be).

      Angelo: Embedded/ extensions or batching with Cypher are your best options to improve performance.

      Regards,

      - Lasse

      Comment

      Working...
      X