From Acegi security 1.0.6 to Spring security 3.2.5 (draft)

It's draft version. I'm going to update it... but most of info already here)

Udgrade main pom.xml

Remove acegi
    <dependencymanagement>
        <dependencies>
         ...
<dependency>
<groupId>org.acegisecurity</groupId>
<artifactId>acegi-security</artifactId>
<version>1.0.6</version>
</dependency>
<dependency>
<groupId>org.acegisecurity</groupId>
<artifactId>acegi-security-tiger</artifactId>
<version>1.0.6</version>
</dependency>
<dependency>
        ...
        </dependencies>

    </dependencymanagement>


Add spring-security

    <properties>
        <spring.version>3.2.8.RELEASE</spring.version>
        <spring.security.version>3.2.5.RELEASE</spring.security.version>
    </properties>
    <dependencymanagement>
        <dependencies>
         ...
            <!--SPRING SECURITY START-->
            <dependency>
                <groupId>org.springframework.security</groupId>
                <artifactId>spring-security-web</artifactId>
                <version>${spring.security.version}</version>
            </dependency>
            <dependency>
                <groupId>org.springframework.security</groupId>
                <artifactId>spring-security-config</artifactId>
                <version>${spring.security.version}</version>
            </dependency>
            <dependency>
                <groupId>org.springframework.security</groupId>
                <artifactId>spring-security-core</artifactId>
                <version>${spring.security.version}</version>
            </dependency>
            <dependency>
                <groupId>org.springframework.security</groupId>
                <artifactId>spring-security-taglibs</artifactId>
                <version>${spring.security.version}</version>
            </dependency>
            <!--SPRING SECURITY END-->
        ...
        </dependencies>

    </dependencymanagement>


Upgrade classes:
org.acegisecurity.GrantedAuthorityImpl -> org.springframework.security.core.authority.SimpleGrantedAuthority

org.acegisecurity.GrantedAuthority  -> org.springframework.security.core.GrantedAuthority
org.acegisecurity.providers.dao.cache.EhCacheBasedUserCache -> org.springframework.security.core.userdetails.cache.EhCacheBasedUserCache;
org.acegisecurity.providers.UsernamePasswordAuthenticationToken -> org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
org.acegisecurity.Authentication -> org.springframework.security.core.Authentication;
org.acegisecurity.context.SecurityContext -> org.springframework.security.core.context.SecurityContext;
org.acegisecurity.context.SecurityContextHolder -> org.springframework.security.core.context.SecurityContextHolder;
org.acegisecurity.AuthenticationException -> org.springframework.security.authentication.AuthenticationServiceException
org.acegisecurity.AuthenticationServiceException -> org.springframework.security.authentication.AuthenticationServiceException
org.acegisecurity.BadCredentialsException -> org.springframework.security.authentication.BadCredentialsException
org.acegisecurity.providers.dao.AbstractUserDetailsAuthenticationProvider -> org.springframework.security.authentication.dao.AbstractUserDetailsAuthenticationProvider
org.acegisecurity.userdetails.UserDetails -> org.springframework.security.core.userdetails.UserDetails
org.acegisecurity.AuthenticationManager -> org.springframework.security.authentication.AuthenticationManager
org.acegisecurity.annotation.Secured -> org.springframework.security.access.annotation.Secured


org.acegisecurity.userdetails.User -> org.springframework.security.core.userdetails.User
for me update constructorur for convert array to collection was enough
class  AppUser {
 public AppUser(String username, String password, boolean enabled, boolean accountNonExpired, boolean credentialsNonExpired,
      boolean accountNonLocked, GrantedAuthority[] authorities) throws IllegalArgumentException {
    super(username, password, enabled, accountNonExpired, credentialsNonExpired, accountNonLocked, Arrays.asList(authorities));
  }
...
}
authorities -> Arrays.asList(authorities)

Upgrade context:

<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xmlns:security="http://www.springframework.org/schema/security"

       xsi:schemaLocation="http://www.springframework.org/schema/beans
                http://www.springframework.org/schema/beans/spring-beans-3.2.xsd
                http://www.springframework.org/schema/security
                http://www.springframework.org/schema/security/spring-security-3.2.xsd">

   <!--was
<bean id="basicProcessingFilter" class="org.acegisecurity.ui.basicauth.BasicProcessingFilter">
      <property name="authenticationEntryPoint" ref="basicProcessingFilterEntryPoint" />
      <property name="authenticationManager" ref="authenticationManager" />
   </bean>
-->
<bean id="springSecurityFilterChain" class="org.springframework.security.web.authentication.www.BasicAuthenticationFilter">
    <property name="authenticationEntryPoint" ref="basicProcessingFilterEntryPoint" />
    <property name="authenticationManager" ref="authenticationManager" />
</bean>

<!--was
   <bean id="basicProcessingFilterEntryPoint" class="org.acegisecurity.ui.basicauth.BasicProcessingFilterEntryPoint">
      <property name="realmName" value="My Realm" />
   </bean>
-->
<bean id="basicProcessingFilterEntryPoint" class="org.springframework.security.web.authentication.www.BasicAuthenticationEntryPoint">
    <property name="realmName" value="My Realm" />
</bean>

<!--
   <bean id="httpSessionContextIntegrationFilter" class="org.acegisecurity.context.HttpSessionContextIntegrationFilter">
      <property name="allowSessionCreation" value="false" />
   </bean>
-->
<bean id="httpSessionContextIntegrationFilter" class="org.springframework.security.web.context.SecurityContextPersistenceFilter">
    <property name="forceEagerSessionCreation" value="false" />
</bean>

<!--was
   <bean id="authenticationManager" class="org.acegisecurity.providers.ProviderManager">
      <property name="providers">
         <list>
            <ref bean="usernamePasswordAuthenticationProvider" />
         </list>
      </property>
   </bean>
-->
<bean id="authenticationManager" class="org.springframework.security.authentication.ProviderManager">
    <property name="providers">
        <list>
            <ref bean="usernamePasswordAuthenticationProvider" />
        </list>
    </property>
</bean>

......
<!--was
   <bean id="roleVoter" class="org.acegisecurity.vote.RoleVoter">
      <property name="rolePrefix" value="" />
   </bean>
-->
<bean id="roleVoter" class="org.springframework.security.access.vote.RoleVoter">
    <property name="rolePrefix" value="" />
</bean>

<!--was
   <bean id="accessDecisionManager" class="org.acegisecurity.vote.AffirmativeBased">
      <property name="allowIfAllAbstainDecisions">
         <value>false</value>
      </property>
      <property name="decisionVoters">
         <list>
            <ref bean="roleVoter" />
         </list>
      </property>
   </bean>
-->
<!--was
   <bean id="securityAnnotationAttributes" class="org.acegisecurity.annotation.SecurityAnnotationAttributes" />

   <bean id="objectDefinitionSource" class="org.acegisecurity.intercept.method.MethodDefinitionAttributes">
      <property name="attributes" ref="securityAnnotationAttributes" />
   </bean>

   <bean id="annotationBasedMethodSecurityInterceptor" class="org.acegisecurity.intercept.method.aopalliance.MethodSecurityInterceptor">
      <property name="validateConfigAttributes" value="false" />
      <property name="authenticationManager" ref="authenticationManager" />
      <property name="accessDecisionManager" ref="accessDecisionManager" />
      <property name="objectDefinitionSource" ref="objectDefinitionSource" />
   </bean>

   <bean class="org.acegisecurity.intercept.method.aopalliance.MethodDefinitionSourceAdvisor">
      <constructor-arg ref="annotationBasedMethodSecurityInterceptor" />
   </bean>
-->

    <security:global-method-security pre-post-annotations="enabled"/>


Upgrade app-servlet.xml
just updated necessary dependency. nathing more.

Upgrade web.xml
<!--was
  <filter>
    <filter-name>Acegi HTTP BASIC Authorization Filter</filter-name>
    <filter-class>org.acegisecurity.util.FilterToBeanProxy</filter-class>
    <init-param>
      <param-name>targetClass</param-name>
      <param-value>org.acegisecurity.ui.basicauth.BasicProcessingFilter</param-value>
    </init-param>
  </filter>

  <filter-mapping>
    <filter-name>Acegi HTTP BASIC Authorization Filter</filter-name>
    <url-pattern>/*</url-pattern>
  </filter-mapping>
-->
    <filter>
        <filter-name>springSecurityFilterChain</filter-name>
        <filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class>
    </filter>
    <filter-mapping>
        <filter-name>springSecurityFilterChain</filter-name>
        <url-pattern>/*</url-pattern>
    </filter-mapping>

Sqoop 2 investigation

I want to share my sqoop2 investigation experience. 

I tried configuring sqoop2 on hortonworks 2.1 VM. I got success result with starting server side, but for some reason I cannot connect client sqoop2 application to server.

I had used 2 instructions:

1.       https://cwiki.apache.org/confluence/display/SQOOP/Sqoop2+Manual+Setup

This one is archived, but firstly I did notice that. I had done everything as in manual and got worked server. Unfortunately manual does not contain client side details.

2.       https://cwiki.apache.org/confluence/display/SQOOP/Sqoop2+Quickstart

This one is latest (Jun 09, 2013), but looks like it is not up to date.

Building, creating binaries, Installing, staring/stopping server parts are correct, but start client and examples is out of day.

I think if I spent some more time I would get success result. But as it’s not main goal I decided work with sqoop2 that installed in latest cloudera VM (hue 5.0).

Also I want notice that web-ui that present in cloudera VM it’s not part of sqoop2 original distributive, it parts of cloudera hue.

Original sqoop2 client is java command line application, but it more user friendly than sqoop 1 interface.

So I downloaded latest cloudera VM from:

http://www.cloudera.com/content/cloudera-content/cloudera-docs/DemoVMs/Cloudera-QuickStart-VM/cloudera_quickstart_vm.html

Based on instruction I had configured sqoop2 there (by default it stopped) and created import sqoop job based on example that they suggested.

 

Some why hue did not run job. I found sqoop2 logs in cloudera VM (they located in folder ‘/var/log/sqoop2’), but I did not find there any information that explain the reason of job failure.

Also I tried configure and run sqoop2 job based on original command line client and got the same issue without any details.

So, my next step was investigating different Hadoop logs and actually finds the reason why it fail.

First interest place is HUE Server logs:

 

 There I found description of error why sqoop2 job had not run. For me it was incorrectly configured sqoop2 connector. I want recommend use ‘http://www.jsoneditoronline.org/’ for analyzing error. As messages from soop2 comes in json format.

BUT, job still did not work.((( 

I did some brain storming and figure out one more important log for sqoop2: http://localhost:8088/cluster/apps

There I at last found the last issue (for me): sqoop2 user has restricted permissions and it cannot write into folder ‘/user/cloudera/emp’

Solution:   change folder ‘/user/sqoop2/emp’

Also I tried sqoop2 java client API it works fine for me, except errors and exceptions they could be more detailed.

Some details that I found about sqoop2 api:

http://sqoop.apache.org/docs/1.99.3/ClientAPI.html

http://sqoop.apache.org/docs/1.99.3/RESTAPI.html

 Summary:

Sqoop 2 looks interesting, but currently it has less functionality then sqoop 1 (for ex.: I did not see how import data into hive as sqoop 1.4.4 can do and so on). Also informative log is also a problem.

So for now I would prefer use sqoop 1.4.4. 

Java decompaling

Java decompiling

 Great tool for decompiling and view code 'Java Decompiler' - http://jd.benow.ca/.

+ Fast decompile
+ Good navigation
- Decompiled code could contain incorrect parts that hard to resolve
- Search in tool could be better)

Fernflower

Great tool for decompiling to improving code. Does not have some site(, but you can download it from link
https://drive.google.com/file/d/0BynTyQpk3OoFc3FwS3lNTE02Qjg/edit?usp=sharing
+ Easy for use
-I have issue with decompiling of class with inner classes, but other stuff decompiled correctly.

References

http://habrahabr.ru/post/176825/
http://pastebin.com/bh83nHL5
https://gist.github.com/agaricusb/4575843

VMWare & CentOS 6.4 Windows X installation

Install X Window

1. Update the installed system
   

   yum -y update

2. Install Desktop and X Window package groups
   

   yum groupinstall "Desctop" "X Window System"

3. Install default/boxes fonts
   

   yum install xorg-x11-fonts-Type1

4. Change default runlevel to 5 (X11 runlevel)
   

   vi /etc/inittab [click insert] id:3:initdefault: -> id:5::initdefault: [click Esc] :wq [click Enter]

Install VMWare drivers

1. Create folder /mnt/cdrom
   

   mkdir /mnt/cdrom

2. Mount cdrom
   

   mount /dev/cdrom1 /mnt/cdrom

3. Get archives with tools and drivers into tmp folder.
   

   cd /mnt/cdrom cp VMwareTools[click tab] /tmp

4. Unpuck and run it
   

   cd /tmp tar -xzvf VMwareTools[tab] cd vmware-tools-distrib/ ./vmware-install.pl [click enter] n-time

Add new user

1. Add new user
   

   useradd bigdata passwd bigdata [set password]

2. Reboot computer
   

   reboot

Sqoop 1.4.3 Create own connector (manager factory)

In case if managers that present in sqoop is not enough you need create your own managers.

Create project

Project should have structure as shown below. You should add to build path sqoop.jar and log4j.jar (in my case: sqoop-1.4.3-SNAPSHOT.jar and commons-logging-1.1.1.jar).

Create new Manager Factory

Create manager factory where you will define your custom managers. It has to extend com.cloudera.sqoop.manager.ManagerFactory. In my case it's CustomManagerFactory.

package sqoop.manager;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sqoop.manager.Db2Manager;
import org.apache.sqoop.manager.DirectMySQLManager;
import org.apache.sqoop.manager.DirectPostgresqlManager;
import org.apache.sqoop.manager.HsqldbManager;
import org.apache.sqoop.manager.OracleManager;
import org.apache.sqoop.manager.PostgresqlManager;
import org.apache.sqoop.manager.SQLServerManager;

import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.manager.ConnManager;
import com.cloudera.sqoop.metastore.JobData;

import com.cloudera.sqoop.manager.ManagerFactory;

public class CustomManagerFactory extends ManagerFactory{

      public static final Log LOG = LogFactory.getLog(
              CustomManagerFactory.class.getName());
      
      public ConnManager accept(JobData data) {
            SqoopOptions options = data.getSqoopOptions();

            String scheme = extractScheme(options);
            if (null == scheme) {
              // We don't know if this is a mysql://, hsql://, etc.
              // Can't do anything with this.
              LOG.warn("Null scheme associated with connect string.");
              return null;
            }

            LOG.debug("Trying with scheme: " + scheme);

            if (scheme.equals("jdbc:mysql:")) {
              if (options.isDirect()) {
                return new DirectMySQLManager(options);
              } else {
                return new MysqlManager(options);
              }
            } else if (scheme.equals("jdbc:postgresql:")) {
              if (options.isDirect()) {
                return new DirectPostgresqlManager(options);
              } else {
                return new PostgresqlManager(options);
              }
            } else if (scheme.startsWith("jdbc:hsqldb:")) {
              return new HsqldbManager(options);
            } else if (scheme.startsWith("jdbc:oracle:")) {
              return new OracleManager(options);
            } else if (scheme.startsWith("jdbc:sqlserver:")) {
              return new SQLServerManager(options);
            } else if (scheme.startsWith("jdbc:db2:")) {
              return new Db2Manager(options);
            } else {
              return null;
            }
          }

          protected String extractScheme(SqoopOptions options) {
            String connectStr = options.getConnectString();

            // java.net.URL follows RFC-2396 literally, which does not allow a ':'
            // character in the scheme component (section 3.1). JDBC connect strings,
            // however, commonly have a multi-scheme addressing system. e.g.,
            // jdbc:mysql://...; so we cannot parse the scheme component via URL
            // objects. Instead, attempt to pull out the scheme as best as we can.

            // First, see if this is of the form [scheme://hostname-and-etc..]
            int schemeStopIdx = connectStr.indexOf("//");
            if (-1 == schemeStopIdx) {
              // If no hostname start marker ("//"), then look for the right-most ':'
              // character.
              schemeStopIdx = connectStr.lastIndexOf(':');
              if (-1 == schemeStopIdx) {
                // Warn that this is nonstandard. But we should be as permissive
                // as possible here and let the ConnectionManagers themselves throw
                // out the connect string if it doesn't make sense to them.
                LOG.warn("Could not determine scheme component of connect string");

                // Use the whole string.
                schemeStopIdx = connectStr.length();
              }
            }
            return connectStr.substring(0, schemeStopIdx);
          }
}

Create custom Manager

Create new manager that will works with data. Manager shoud extend org.apache.sqoop.manager.InformationShemaManager

package sqoop.manager;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sqoop.manager.*;
import com.cloudera.sqoop.SqoopOptions;
public class MysqlManager extends InformationSchemaManager {
    
    public static final Log LOG = LogFactory.getLog(MysqlManager.class.getName());
    private static final String STRING = "";
    public MysqlManager(SqoopOptions opts) {
        super(STRING, opts);
    }
    @Override
    protected String getSchemaQuery() {
        return null;
    }
    @Override
    protected String getListDatabasesQuery() {
        return null;
    }
}

Configure sqoop for work with external factory manager

First of all you have to define SQOOP_CONF_DIR for example '/home/cloudera/sqoop/conf'. After that in this folder you have to create folder 'managers.d'. In this folder you have to create property file, for example 'managers.property'.

In this file you can define you own factory manager. Structure of file you can see below.

sqoop.manager.CustomManagerFactory=/home/cloudera/workspace/extention-conn-manager/manager.jar

After next start of sqoop you have to see defined your factory manager, as shown below.

Example Project

https://drive.google.com/#folders/0BynTyQpk3OoFNkg0dnJaWHFlVXc