Counting greps

 #!/bin/bash
 
if [ $# -eq 1 ]
  then
   echo "file log: $1 - num of occurrences: "
   grep -w "string to grep" $1 | wc -l
else
 echo "no args specified: insert a file name"
fi

Monitoring directory to handle excel files in python

 My last post was about how to remove columns from a excel in python. I needed to monitor a directory so I did it directly in python, here is the result: handle columns in a excel file in every file added to a directory.

import time
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import shutil
import pandas as pd
import sys
 
class Watcher:
        DIRECTORY_TO_WATCH =  "Test"
 
        def __init__(self):
            self.observer = Observer()
 
        def run(self):
            event_handler = Handler()
            self.observer.schedule(event_handler, self.DIRECTORY_TO_WATCH, recursive=True)
            self.observer.start()
            try:
                while True:
                    time.sleep(5)
            except:
                    self.observer.stop()
                    print("Error")
 
            self.observer.join()
 
class Handler(FileSystemEventHandler):
 
    @staticmethod
    def on_any_event(event):
        if event.is_directory:
            return None
 
        elif event.event_type == 'created':
            print( "Received %s." % event.src_path)
            fileToHandle = event.src_path
            print(fileToHandle)
            df = pd.read_excel(fileToHandle)
            print(df)
            df = df.drop(
                ['IDCONTRATTO', 'IDHARDWARE', 'DATAEVENTO'], axis=1)
            print(df)
 
            filename = fileToHandle.split("/")
            outputFile = filename[1]
            df.to_excel(outputFile)
            shutil.move(outputFile, "OutputTest")  # todo sostituire con il nome della dir.
 
 
if __name__ == '__main__':
    w = Watcher()
    w.run()

Remove multiple columns from exel with Pandas in Python

import pandas as pd
import xlwt 

df = pd.read_excel('input.xls')
print(df)
 
df = df.drop(['IDCONTRATTO', 'IDHARDWARE', 'DATAEVENTO'], axis=1)
 
print(df)
 
df.to_excel('output.xls')
 

 

 

-----------------------------

 # Then I edited the script to get param from a Input folder and move it to a "OutDir" folder. (just because I needed this, it sucks but it is the way I had to do it.)

import shutil 
import pandas as pd
import sys
import xlwt
 
inputFile = str(sys.argv[1])
print(inputFile)
 
df = pd.read_excel(inputFile)
 
print(df)
 
df = df.drop(['IDCONTRATTO', 'IDHARDWARE', 'DATAEVENTO'], axis=1)
 
print(df)
 
filename = inputFile.split("/")
outputFile = filename[1]
df.to_excel(outputFile)
 
shutil.move(outputFile,"OutDir")

JWT token based REST API

 
 
 
 
@Data
public class JWTData {
    String accessToken;
    long expiresIn;
    String tokenType;
    String scope;
}
 
------------------------------------------------------------------------------------------------------------
 
public void sendRequest() throws Exception {
 
 final String jwturi = "http://someurl";
 RestTemplate JWTRestTemplate = new RestTemplate();
 HttpHeaders tokenheaders = new HttpHeaders();
 tokenheaders.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
 
 MultiValueMap<String, String> jwtParameters= new LinkedMultiValueMap<>();
 jwtParameters.add("grant_type", "client_credentials");
 jwtParameters.add("client_id", "jwtClientId");
 jwtParameters.add("client_secret", "jwtClientSecret");
 HttpEntity<MultiValueMap<String, String>> tokenRequest = new HttpEntity<>(jwtParameters, tokenheaders);
 ResponseEntity<JWTData> response = JWTRestTemplate.postForEntity(jwturi, tokenRequest , JWTData.class);
 log.info("\n\n-- JWT RESPONSE BODY: {} ", response.getBody().toString());
 String JWT = response.getBody().getAccessToken();
 log.info("access token: " + JWT);
 
 String alertDataAsString = jsonMapper.writeValueAsString(alertData);
 final String uri = alarmNotificationUrl;
 RestTemplate restTemplate = new RestTemplate();
 restTemplate.setErrorHandler(new MyRestErrorHandler());
 HttpHeaders headers = new HttpHeaders();
 headers.setContentType(MediaType.APPLICATION_JSON);
 headers.add("Authorization", "Bearer access_token"+ JWT);
 log.info("Json sent: " + alertDataAsString);
 HttpEntity<?> request = new HttpEntity<Object>(alertDataAsString, headers);
 ResponseEntity<Object> responseAsString = restTemplate.exchange(uri, HttpMethod.POST, request, Object.class);
 log.info("\n\n-- RESPONSE STATUS CODE: {} ", responseAsString.getStatusCode());
 log.info("\n\n-- RESPONSE BODY: {} ", responseAsString.getBody().toString());
}
 
 
------------------------------------------------------------------------------------------------------------
 
@Slf4j
public class MyRestErrorHandler extends DefaultResponseErrorHandler {
    @Override
    public void handleError(ClientHttpResponse response) throws IOException {
        log.info("Errore in chiamata agli endpoint");
    }
}

SSO With Keycloak and Liferay

My intent is to configure SSO on Keycloak and Liferay.
I have createad a docker-compose environment  with Keycloak:

################################

version: '3.7'
services:
  mysql:
    image: mysql:5.7.29
    container_name: mysql-SSO
    ports:
      - "3306:3306"
    environment:
      - MYSQL_DATABASE=keycloak
      - MYSQL_USER=keycloak
      - MYSQL_PASSWORD=password
      - MYSQL_ROOT_PASSWORD=root_password
    healthcheck:
      test: "mysqladmin ping -u root -p$${MYSQL_ROOT_PASSWORD}"
      start_period: 10s

  keycloak:
    image: jboss/keycloak:9.0.3
    container_name: keycloak-SSO
    environment:
      - KEYCLOAK_USER=admin
      - KEYCLOAK_PASSWORD=admin
      - DB_VENDOR=mysql
      - DB_ADDR=mysql
      - DB_USER=keycloak
      - DB_PASSWORD=password
      - JDBC_PARAMS=useSSL=false
    ports:
      - 8080:8080
    depends_on:
      - mysql
    healthcheck:
      test: "curl -f http://localhost:8080/auth || exit 1"
      start_period: 10s

################################

At first I tryed to configure SSO using a Liferay Docker image:

docker run --name='liferay' -d -p 8081:8080 liferay/portal:7.1.0-ga1-201809012030

 

Unfortunately I have a redirection error on liferay when it receives the auth token from Keycloak. So I decided to give a test with a real instance of Liferay, so I installed it and started Liferay.


I changed Liferay Tomcat port to 8090.

So I did the following cofiguration steps:

on Keycloak:
  

• Realm Settings: click on Endpoints "OpenId Endpoint Configuration" and leave the tab opened, we well need those values in a while.
• Clients: Create new Client

 

• Credentials: copy client secrets.  

• Identity Providers: Create a new idp: Keycloak OpenID Connect, use the tab we left open at step 1 to fill URLS and choose "Client secrets sent as basic auth" on Client Authentication, and the client-id has to be the same of the one configured before.
• Users: create a user, and set passwords in Credentials.
 

On Liferay:

• Configuration -> System Settings -> SSO -> OpenIdConnect: Check Enabled  
• Configuration -> System Settings -> SSO -> OpenId Connect Provider: create new provider:
• provider name: keycloak idp
• secret: keycloak clients secret
• Provider Name: from Keycloak
• OpenID Connect Client ID: from Keycloak
• OpenID connect client secret: from Keycloak
• Scopes: openid email profile
• Discovery Endpoint Cache in Milliseconds: 360000
• Authorization Endpoint: http://localhost:8080/auth/realms/master/protocol/openid-connect/auth
• Issuer URL: http://localhost:8080/auth/realms/master
• JWKS URI: http://localhost:8080/auth/realms/master/protocol/openid-connect/certs
• ID Token Signing Algorithms: RS256
• Subject Types: public
• Token Endpoint: http://localhost:8080/auth/realms/master/protocol/openid-connect/token
• User Information Endpoint: http://localhost:8080/auth/realms/master/protocol/openid-connect/userinfo
• Go to anonymous page and click on sign in. Then click on OpenId Connect.
• Login with the user created in Keycloak and pray.

Once signed in on Liferay through OpenIdConnect Liferay could require email verification.
If it happens we have to disable the functionality in authentication settings.

HowTo: custom settings.xml with secrets in CI/CD

I have to deploy to Apache Archiva with a custom .m2/settings.xml in a Gitlab project. So I am doing the following 3 steps:

1. my .gitlab-ci.yml

stages:
  - build

Build:
  stage: build
  image: maven:latest
  script:
    - mvn --settings .m2/settings.xml -DskipTests=true clean package


2. my env variables in Gitlab CI/CD Settings:

3. custom .m2/settings.xml in project:

[...]
        <server>
            <id>archiva.snapshots</id>
            <username>${ENV.MAVEN_USER}</username>
            <password>${ENV.MAVEN_PASSWORD}</password>
        </server>
[...]

Presentazione introduttiva alla CI/CD

Una breve presentazione introduttiva che ho dovuto fare per un team di engineering di un'azienda. Mostra l'utilizzo delle pipeline, alcuni esempi di CI/CD, cenni su GitLab e il continuous integration delivery e deployment e un modus operandi utile allo sviluppo collaborativo di progetti mirato all'automatizzazione del mantenimento, testing e rilascio del software.

Anche se senza una voce narrante perdono di logica, quello che ho voluto mostrare in queste slide e' la potenza che ha un software con una componente di CI/CD nell'automatizzare le fasi di build test e rilasci senza perdere tempo e risorse sul mantenimento dello stesso.

Questo e' possibile solo attraverso una giusta ed equilibrata collaborazione tra sviluppatori e DevOps, affiancata da adeguate metodologie agili.

Camel Kafka consumer/producer

This is a simple integration I used with springboot, camel, kafka. I had to consume from a topic and do some logic to produce on another topic. I followed the same idea to develop a "topic switcher" that chose on what topic to produce (not in this post but the solution is immediate).
The routebuilder is the main camel class that call "getBody" method on a bean that do the magic. The bean call back the routebuilder to produce on Kafka.



//ROUTEBUILDER:

@Component
public class MyRouteBuilder extends RouteBuilder {

    @Override
    public void configure() throws Exception {

        from("kafka:{{consumer.topic}}?brokers={{kafka.brokers}}&consumersCount={{consumer.consumersCount}}&groupId={{consumer.group}}")
                .routeId("FromKafka")
                .log("------- ROUTE BUILDER -------")
                .onException(Exception.class)
                .log(LoggingLevel.ERROR, "crash-data-stream", "Invalid Input")
                .maximumRedeliveries(2).redeliveryDelay(30000)
                .end()
                .to("bean:externalBean?method=getBody(${body})");

        from("direct:myproducer")
                .routeId("myproducer")
                .to("kafka:{{producer.topic}}?brokers={{kafka.brokers}}")
                ;
    }
}


----------------------------------

//EXTERNAL BEAN

@Service
public class ExternalBean {

    @EndpointInject("direct:myproducer")
    ProducerTemplate producer;
 
    public void getBody(String body) throws Exception {
        //do stuff and send back to camel
        myproducer.sendBody(body);
    }

}

Spring boot multiple property files

Add in pom:

<build>

...

  <resources>

   <resource>

    <directory>src/main/resources</directory>

    <filtering>true</filtering>

   </resource>

  </resources>

...

</build>

2. Add

<plugin>

  <groupId>org.apache.maven.plugins</groupId>

   <artifactId>maven-help-plugin</artifactId>

    <version>3.2.0</version>

     <executions>

      <execution>

       <id>show-profiles</id>

        <phase>compile</phase>

        <goals>

         <goal>active-profiles</goal>

        </goals>

      </execution>

     </executions>

</plugin>

3. Add

<profiles>

  <profile>

   <id>dev</id>

    <properties>

     <activatedProperties>dev</activatedProperties>

    </properties>

    <activation>

     <activeByDefault>true</activeByDefault>

    </activation>

  </profile>

  <profile>

  <id>prod</id>

   <properties>

     <activatedProperties>prod</activatedProperties>

   </properties>

  </profile>

</profiles>

4. Create application.properties:

spring.profiles.active=@activatedProperties@

5. Create:

application-dev.properties

application-prod.properties

6. run:

mvn clean spring-boot:run -Pprod

//or

mvn clean spring-boot:run -Dspring-boot.run.profiles=dev

Install RabbitMq on Centos

sudo yum install epel-release
yum update
wget https://packages.erlang-solutions.com/erlang-solutions-1.0-1.noarch.rpm
ls
rpm -Uvh erlang-solutions-1.0-1.noarch.rpm
yum install erlang
yum update
yum install erlang
yum remove erlang-erts
yum install erlang
ls
rpm --install rabbitmq-server-3.8.1-1.el8.noarch.rpm
yum info rabbitmq-server
service rabbitmq-server status
service rabbitmq-server start
service rabbitmq-server status