$ minikube delete
$ minikube delete --all --purge

$ minikube start --wait=false

Kubectl

kubectl cluster-info

k8s docs

Context and configuration

$ kubectl config view # Show Merged kubeconfig settings.

$ kubectl config view -o jsonpath='{.users[].name}'    # display the first user
$ kubectl config view -o jsonpath='{.users[*].name}'   # get a list of users
$ kubectl config get-contexts                          # display list of contexts 
$ kubectl config current-context                       # display the current-context
$ kubectl config use-context my-cluster-name           # set the default context to my-cluster-name

Apply

Viewing, finding resources

# Get commands with basic output
kubectl get services                          # List all services in the namespace
kubectl get pods --all-namespaces             # List all pods in all namespaces
kubectl get pods -o wide                      # List all pods in the current namespace, with more details
kubectl get deployment my-dep                 # List a particular deployment
kubectl get pods                              # List all pods in the namespace
kubectl get pod my-pod -o yaml                # Get a pod's YAML

# Describe commands with verbose output
kubectl describe nodes my-node
kubectl describe pods my-pod

# List Services Sorted by Name
kubectl get services --sort-by=.metadata.name

# List pods Sorted by Restart Count
kubectl get pods --sort-by='.status.containerStatuses[0].restartCount'

# List PersistentVolumes sorted by capacity
kubectl get pv --sort-by=.spec.capacity.storage

# Get the version label of all pods with label app=cassandra
kubectl get pods --selector=app=cassandra -o \
  jsonpath='{.items[*].metadata.labels.version}'

# Retrieve the value of a key with dots, e.g. 'ca.crt'
kubectl get configmap myconfig \
  -o jsonpath='{.data.ca\.crt}'

# Retrieve a base64 encoded value with dashes instead of underscores.
kubectl get secret my-secret --template=''

# Get all worker nodes (use a selector to exclude results that have a label
# named 'node-role.kubernetes.io/control-plane')
kubectl get node --selector='!node-role.kubernetes.io/control-plane'

# Get all running pods in the namespace
kubectl get pods --field-selector=status.phase=Running

# Get ExternalIPs of all nodes
kubectl get nodes -o jsonpath='{.items[*].status.addresses[?(@.type=="ExternalIP")].address}'

# List Names of Pods that belong to Particular RC
# "jq" command useful for transformations that are too complex for jsonpath, it can be found at https://stedolan.github.io/jq/
sel=${$(kubectl get rc my-rc --output=json | jq -j '.spec.selector | to_entries | .[] | "\(.key)=\(.value),"')%?}
echo $(kubectl get pods --selector=$sel --output=jsonpath={.items..metadata.name})

# Show labels for all pods (or any other Kubernetes object that supports labelling)
kubectl get pods --show-labels

# Check which nodes are ready
JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' \
 && kubectl get nodes -o jsonpath="$JSONPATH" | grep "Ready=True"

# Output decoded secrets without external tools
kubectl get secret my-secret -o go-template='### \n\n\n'

# List all Secrets currently in use by a pod
kubectl get pods -o json | jq '.items[].spec.containers[].env[]?.valueFrom.secretKeyRef.name' | grep -v null | sort | uniq

# List all containerIDs of initContainer of all pods
# Helpful when cleaning up stopped containers, while avoiding removal of initContainers.
kubectl get pods --all-namespaces -o jsonpath='{range .items[*].status.initContainerStatuses[*]}{.containerID}{"\n"}{end}' | cut -d/ -f3

# List Events sorted by timestamp
kubectl get events --sort-by=.metadata.creationTimestamp

# List all warning events
kubectl events --types=Warning

# Compares the current state of the cluster against the state that the cluster would be in if the manifest was applied.
kubectl diff -f ./my-manifest.yaml

# Produce a period-delimited tree of all keys returned for nodes
# Helpful when locating a key within a complex nested JSON structure
kubectl get nodes -o json | jq -c 'paths|join(".")'

# Produce a period-delimited tree of all keys returned for pods, etc
kubectl get pods -o json | jq -c 'paths|join(".")'

# Produce ENV for all pods, assuming you have a default container for the pods, default namespace and the `env` command is supported.
# Helpful when running any supported command across all pods, not just `env`
for pod in $(kubectl get po --output=jsonpath={.items..metadata.name}); do echo $pod && kubectl exec -it $pod -- env; done

# Get a deployment's status subresource
kubectl get deployment nginx-deployment --subresource=status

Updating resources

kubectl set image deployment/frontend www=image:v2               # Rolling update "www" containers of "frontend" deployment, updating the image
kubectl rollout history deployment/frontend                      # Check the history of deployments including the revision
kubectl rollout undo deployment/frontend                         # Rollback to the previous deployment
kubectl rollout undo deployment/frontend --to-revision=2         # Rollback to a specific revision
kubectl rollout status -w deployment/frontend                    # Watch rolling update status of "frontend" deployment until completion
kubectl rollout restart deployment/frontend                      # Rolling restart of the "frontend" deployment


cat pod.json | kubectl replace -f -                              # Replace a pod based on the JSON passed into stdin

# Force replace, delete and then re-create the resource. Will cause a service outage.
kubectl replace --force -f ./pod.json

# Create a service for a replicated nginx, which serves on port 80 and connects to the containers on port 8000
kubectl expose rc nginx --port=80 --target-port=8000

# Update a single-container pod's image version (tag) to v4
kubectl get pod mypod -o yaml | sed 's/\(image: myimage\):.*$/\1:v4/' | kubectl replace -f -

kubectl label pods my-pod new-label=awesome                      # Add a Label
kubectl label pods my-pod new-label-                             # Remove a label
kubectl annotate pods my-pod icon-url=http://goo.gl/XXBTWq       # Add an annotation
kubectl autoscale deployment foo --min=2 --max=10                # Auto scale a deployment "foo"

Patching resources

# Partially update a node
kubectl patch node k8s-node-1 -p '{"spec":{"unschedulable":true}}'

# Update a container's image; spec.containers[*].name is required because it's a merge key
kubectl patch pod valid-pod -p '{"spec":{"containers":[{"name":"kubernetes-serve-hostname","image":"new image"}]}}'

# Update a container's image using a json patch with positional arrays
kubectl patch pod valid-pod --type='json' -p='[{"op": "replace", "path": "/spec/containers/0/image", "value":"new image"}]'

# Disable a deployment livenessProbe using a json patch with positional arrays
kubectl patch deployment valid-deployment  --type json   -p='[{"op": "remove", "path": "/spec/template/spec/containers/0/livenessProbe"}]'

# Add a new element to a positional array
kubectl patch sa default --type='json' -p='[{"op": "add", "path": "/secrets/1", "value": {"name": "whatever" } }]'

# Update a deployment's replica count by patching its scale subresource
kubectl patch deployment nginx-deployment --subresource='scale' --type='merge' -p '{"spec":{"replicas":2}}'

Scaling resources

kubectl scale --replicas=3 rs/foo                                 # Scale a replicaset named 'foo' to 3
kubectl scale --replicas=3 -f foo.yaml                            # Scale a resource specified in "foo.yaml" to 3
kubectl scale --current-replicas=2 --replicas=3 deployment/mysql  # If the deployment named mysql's current size is 2, scale mysql to 3
kubectl scale --replicas=5 rc/foo rc/bar rc/baz                   # Scale multiple replication controllers

Deleting resources

kubectl delete -f ./pod.json                                      # Delete a pod using the type and name specified in pod.json
kubectl delete pod unwanted --now                                 # Delete a pod with no grace period
kubectl delete pod,service baz foo                                # Delete pods and services with same names "baz" and "foo"
kubectl delete pods,services -l name=myLabel                      # Delete pods and services with label name=myLabel
kubectl -n my-ns delete pod,svc --all                             # Delete all pods and services in namespace my-ns,
# Delete all pods matching the awk pattern1 or pattern2
kubectl get pods  -n mynamespace --no-headers=true | awk '/pattern1|pattern2/{print $1}' | xargs  kubectl delete -n mynamespace pod

Interacting with running Pods

kubectl logs my-pod                                 # dump pod logs (stdout)
kubectl logs -l name=myLabel                        # dump pod logs, with label name=myLabel (stdout)
kubectl logs my-pod --previous                      # dump pod logs (stdout) for a previous instantiation of a container
kubectl logs my-pod -c my-container                 # dump pod container logs (stdout, multi-container case)
kubectl logs -l name=myLabel -c my-container        # dump pod logs, with label name=myLabel (stdout)
kubectl logs my-pod -c my-container --previous      # dump pod container logs (stdout, multi-container case) for a previous instantiation of a container
kubectl logs -f my-pod                              # stream pod logs (stdout)
kubectl logs -f my-pod -c my-container              # stream pod container logs (stdout, multi-container case)
kubectl logs -f -l name=myLabel --all-containers    # stream all pods logs with label name=myLabel (stdout)
kubectl run -i --tty busybox --image=busybox:1.28 -- sh  # Run pod as interactive shell
kubectl run nginx --image=nginx -n mynamespace      # Start a single instance of nginx pod in the namespace of mynamespace
kubectl run nginx --image=nginx --dry-run=client -o yaml > pod.yaml
                                                    # Generate spec for running pod nginx and write it into a file called pod.yaml
kubectl attach my-pod -i                            # Attach to Running Container
kubectl port-forward my-pod 5000:6000               # Listen on port 5000 on the local machine and forward to port 6000 on my-pod
kubectl exec my-pod -- ls /                         # Run command in existing pod (1 container case)
kubectl exec --stdin --tty my-pod -- /bin/sh        # Interactive shell access to a running pod (1 container case)
kubectl exec my-pod -c my-container -- ls /         # Run command in existing pod (multi-container case)
kubectl top pod POD_NAME --containers               # Show metrics for a given pod and its containers
kubectl top pod POD_NAME --sort-by=cpu              # Show metrics for a given pod and sort it by 'cpu' or 'memory'

Copy files and directories to and from containers

kubectl cp /tmp/foo_dir my-pod:/tmp/bar_dir            # Copy /tmp/foo_dir local directory to /tmp/bar_dir in a remote pod in the current namespace
kubectl cp /tmp/foo my-pod:/tmp/bar -c my-container    # Copy /tmp/foo local file to /tmp/bar in a remote pod in a specific container
kubectl cp /tmp/foo my-namespace/my-pod:/tmp/bar       # Copy /tmp/foo local file to /tmp/bar in a remote pod in namespace my-namespace
kubectl cp my-namespace/my-pod:/tmp/foo /tmp/bar       # Copy /tmp/foo from a remote pod to /tmp/bar locally

Jonas is 28 years old and an aspiring heart surgeon. Some would argue that at 28, he’s still a bit young to be a heart surgeon. But Jonas has always been ahead of his time and is very talented. In his younger years, he was already present at many of his colleagues’ surgeries, was allowed to assist and already had some of these vital organs in his hands.

In fact, they were all different, but as Jonas looked closer, he saw many similarities. Well, at least they were all hearts, you might think. But let him tell us about his discovery himself:

I have always been fascinated by the most important organ in our body. The heart looks different with each person. Sometimes a little smaller, sometimes larger. Sometimes, unfortunately, the heart does not work as it should. When I looked very closely, I could even see broken hearts.

Some of the people who needed heart surgery seemed to work directly in the IT industry. Which is why I tried to find a connex.

Teams act like hearts, I thought to myself. They keep our organization, our organism, alive. Without a heart, we could not survive and so could not our organization.

Communication is like an artery through which our heart is supplied with oxygen. If our arteries are clogged, our heart doesn’t work properly either - in the worst cases we die!

Our communication, our teams and if we think it further our culture, do not only determine the survival of our organization, but wrongly treated they make us stop functioning or even ceasing to exist from one moment to the next!

Dr. Jonas and his colleagues

Jonas has had an uneasy feeling for some time and has been looking for answers to his thesis in the IT industry. He recognizes many hearts that are seriously ill. The COVID-19 year, with its #WFA (work from anywhere) or the change in face-to-face communication, was in some places just a drop in the bucket. He comes to one conclusion:

The IT industry is in the midst of a depression, and not just since yesterday. It has been shown to us more clearly than ever what many of us are facing.

Why is it not our highest priority to keep our heart healthy?

Heart attack - I hope not

With these findings, Jonas goes to his experienced colleagues, with whom he has always had a very good exchange and for whom the innovative views he brings in are a benefit. He shows them what he has discovered, which startles his colleagues, who thank him for this new insight. After careful consideration, his colleagues recognize a pattern:

It seems that we have put our bodies through a lot in recent years. In many IT organizations, the sinus rhythm does not seem to match the ideal, we are ailing. Have we done too little sport or been out in the fresh air too rarely? In times of Covid-19, this could well be a reason. In addition, there are the psychological burdens to which we are exposed during what feels like our 100th lockdown.

We need to save our organization from a heart attack. Constant movement and adaptation protects us from this … and of course all the good characters from Once upon a time … Life

Once upon a time … Life

Keep moving

Why do we still have such a hard time adapting to change despite the agile manifesto with the value Responding to change that has been known for 20 years? Have we forgotten how to adapt? Have we become too inert and do we reflect far too rarely on what has happened? Jonas wonders why the IT pandemic is hitting us so hard. And he is not talking about the disease itself.

IT organizations today should be able to react very fast and flexible to changes in their market. A customer-centric mindset helps us significantly in this regard. But what Jonas could observe just last year is that the changes are hardly directed inwards. Working methodologies, way of working, processes, teams, communication channels … many things simply remain unchanged.

Keep moving

Jonas remembers a tweet he read just a while ago. It was about a product development and the organizational goals for the next year:

C-Level to PM: We need to roll out 20 new projects this year to be successful, but the development team believes they could only focus on 2 projects at a time with the best quality. What should we do?

This did not sound at all like healthy growth to him. The correct answer to the question, as he found it, and whether this was really done in reality remains to be seen, was:

PM to C-Level: We look for the 2 projects with the highest priority, the greatest customer value and the best return on investment. And focus entirely on these projects and drop the others!

Of course, it is a major challenge to explain this situation to the C-level management, but what would be the alternative? External contractors, outsourcing projects … I hope you know where I am going with this.

• Organic growth is healthy
• Change to be able to react to market situations is good
• Intrinsic adaptation at the same speed is effective

Maybe you know similar situations or are currently in a similar job environment. In an organization that has grown very quickly and seems to struggle with structures and processes.

One body - One life

Jonas continues to reflect:

Our heart, as we know, can perfectly supply a body for years. In some exceptional cases, it is even strong enough to keep 2 bodies alive, as in the case of Siamese twins. I, as an aspiring heart surgeon, believe I can claim that this only works in certain and very special cases and only for a certain period of time.

If we now compare a delivery team in our IT organization with a body, a delivery team should, just like the heart, focus on one major topic, one business domain, one bounded context or one value stream, and master it perfectly.

This is how we create a continuous flow and master our business best. However, if a team has to serve multiple domains, or in other words, if the heart suddenly had to take over the tasks of another organ, the cognitive load in the team increases tremendously.

Hardly feasible, as Jonas feels - The focus is lost, we suffer from loss of quality. It is not one of our main objectives. Little by little, the team will slow down, the quality will move into the negative. We will pay for our misguided decisions.

Focus on excellence

If we stay with IT organizations, we talk about establishing feature teams and cross-functional teams. So we want to make sure that all the necessary skills are in a delivery team or in an extended delivery team. We are looking for motivated contributors who are not only experts in their field, but also think outside their comfort zone (T-shaped). If this alone would not be a huge challenge, there are other aspects that we must not neglect under any circumstances.

We have to take diversity into account in our teams, and thus bring different points of view into the team and, above all, we also promote creativity. We should also not disregard the different and preferably homogeneous seniority levels.

Focus on excellence

We have to make sure that we create long-lasting teams that encourage, support and challenge each other at the right moment, always with the appropriate professionalism, but also with the highest possible trust - without trust we don’t create teams, we create waste! We want to create high-performing teams and thus a results-oriented organization.

This list is far from complete, but it should show that the composition of one or more teams and team topologies is a very complex and extensive topic.

But back to Jonas.

So we now know that continuous change helps us prevent heart attacks. Furthermore, we have found that focus is an important aspect and that we need to avoid cognitive load. And we know that the way we communicate, among other things, determines success or failure.

Can we use these findings to prevent the whole thing from happening?

In my early days as an apprentice in the IT industry, as a software developer, later as a software architect and even as a team leader, I often wondered why it was so difficult for us to fulfill the wishes of management. Only later I realized that at that time probably nobody could have fulfilled them, because the wishes changed from day to day. At that time, management didn’t know what was important to them tomorrow, or they just couldn’t communicate it. In the beginning, I was still fooled into thinking that this is what agile meant - being able to respond to management’s wishes at any point in time. Who was I to question that. I didn’t visit university, I learned the theory in practice. Possibly I just didn’t understand the theory correctly.

It chased me and just wouldn’t leave me alone.

I guess it’s just the way it is. There’s nothing you can do about it! Or can you?

I began to question the theory and wanted to see for myself what is possible. One of the first books I read on leadership was ‘Managing Humans - Biting and Humorous Tales of a Software Engineering Manager’, by Michael Lopp. That was the first time, things seemed so clear to me. Much of what I have read in this book seemed familiar. On the one hand it was amusing that not only I have had this kind of experience, on the other hand it was shocking that there was such a kind of literature available for everyone who is even remotely interested in leadership - nevertheless we still go on like this.

I have to admit that I don’t know if I read it in one of the great books on leadership or if it just evolved over the years and many, many conversations with colleagues, but it is a quote that has stuck with me ever since:

It’s not about managing people, it’s about managing the environment that surrounds them.

Manage the environment

So what is the environment we are referring to here? I would say it is the team area, the system, the organization in general. The idea is to have a serve-first mindset and focus on empowering and developing people. We actively seek to develop and align an individual’s sense of purpose with the business goals. Servant leaders remove organizational impediments, in other words blockers. Do you have everything you need to get the job done? In other words, servant leaders clear the way for others to contribute.

After many more articles, blogs, and a few more fantastic books, I began to understand that the whole topic of leadership is probably a bit more complex:

• ‘The Five Dysfunctions of a Team: A Leadership Fable’, by Patrick Lencioni
• ‘Management 3.0: Leading Agile Developers, Developing Agile Leaders’, by Jurgen Appelo
• ‘Story Driven: You don’t need to compete when you know who you are’, by Bernadette Jiwa

I have tried to apply, try out and adapt what I have learned in different positions and different IT organizations. Some things worked well, some things didn’t.

But what has always worked was to communicate in a way that is polite, inspiring and motivating, even in an uncertain or changing environment. When responding with consideration to colleagues’ personal needs and feelings. When recognizing, valuing and communicating the achievement of goals.

Build trust

As a manager, you often seem untouchable to many employees. Some managers are hardly tangible for their employees. Their door is usually locked. Managers only leave their offices when the roof is on fire, at least that is how it appears to many.

Trust, but verify.

used by Lenin

So it’s not surprising that in many companies you notice as soon as managers take part in a meeting. It seems as if managers give a monologue - employees are not given the opportunity to contribute. They don’t question anything and withdraw.

At the beginning of my career I reacted similarly, then as a team leader I saw the other side and wondered why others reacted to me in the same way. There was a lack of trust, from both sides - at least that was the feeling at the time. We worked on it and it was great to see how the team grew together. To make it even more explicit, trust is always better than control!

In Lencioni’s pyramid, trust is the basis for everything, and I see it the same way. For building trust in your team culture, create an environment where you are open and honest, share problems, admit mistakes, and support each other.

A good first step for managers is to …

Don’t be a prick - Go first - Be human! Make yourself vulnerable.

But of course there were also conflicts in the team that we had to solve here and there.

Give a voice

Welcome productive conflict in your team. Encourage others to speak up, get input from the team and confront issues quickly. But remember, productive and constructive conflict cannot happen without trust!

With trust and healthy conflicts, you engage your team. This ensures that you are aligned on common objectives and being clear on direction and priorities. Team members should feel that they can take responsibility and hold each other accountable, even when it is difficult. Similarly, a high-performing team also recognizes the performance of others with praise or rewards.

Voice has a positive impact on organizational culture and is fostered by giving your teams more autonomy and trust.

Do retrospectives

Retrospectives, learning reviews, and post-mortems are team-based activities designed to learn systematically from past events in order to improve future performance. Delivery teams carry out retrospectives frequently and perform post-mortems or learning reviews after incidents.

As a developer myself, I have experienced post-mortems when errors occur in the system. Most of the time, the management started like this “Who caused this error again?”. Nice, isn’t it? Later I have seen that retrospectives were conducted, but the outcome was equally questionable. “What’s the point in doing retrospectives? It’s just a waste of time and I could use the time for something meaningful, like development.”, I have heard team members say.

Of course, these were not post-mortems or good retrospectives, but a pure blame-game without any meaningful learning.

Both the agile and operations communities emphasize the importance of making these activities blameless. Norman L. Kerth in ‘Project Retrospectives: A Handbook for Team Reviews’ proposes the idea of Retrospective Prime Directive which every participant should read at the beginning of a retrospective.

Retrospective Prime Directive

Regardless of what we discover, we must understand and truly believe that everyone did the best job they could, given what was known at the time, their skills and abilities, the resources available, and the situation at hand.

Norman L. Kerth, 2001

Learning reviews and retrospectives help contribute to a climate for learning, and also impact organizational culture. When teams conduct these reviews and retrospectives, they learn from their mistakes and failures and turn them into opportunities to improve their way of working.

In particular, teams that leverage their findings to implement changes to tooling, processes, or procedures see the strongest impacts.

For an in-depth look at learning reviews and retrospectives, see the 2018 State of DevOps Report.

Be transformational

Some of the challenges mentioned above have led me again and again to the Transformational leadership’s model. It combines many good aspects of leadership. The idea of the Transformational leadership model goes back to James MacGregor Burns in the ’70s. Burns described the difference between transactional and transformational leadership at that time.

Despite the great interest in transformational leadership, a number of theoretical problems have been identified with this model.

In the study Dimensions of transformational leadership: conceptual and empirical extensions (2004), published by Mark A. Griffin and Alannah E. Rafferty, aspects of transformational leadership theory were identified that led to a lack of empirical support for the hypothesized factor structure of the model and very strong relationships among leadership components.

They proposed five more focused subdimensions of transformational leadership, including vision, inspirational communication, intellectual stimulation, supportive leadership and personal recognition. Without a vision, we don’t know where to go. Lewis Carroll paraphrased this beautifully in Alice’s Adventures in Wonderland.

Alice: Would you tell me, please, which way I ought to go from here?
The Cheshire Cat: That depends a good deal on where you want to get to.
Alice: I don’t much care where.
The Cheshire Cat: Then it doesn’t much matter which way you go.
Alice: … So long as I get somewhere.
The Cheshire Cat: Oh, you’re sure to do that, if only you walk long enough.

Lewis Carroll, 1865

Transformational leadership is a model in which leaders inspire and motivate followers to achieve higher performance by appealing to their values and sense of purpose, facilitating wide-scale organizational change. These leaders encourage their teams to work towards a common goal through their vision, values, communication, example-setting, and their evident caring about their followers’ personal needs.

It has been observed that there are similarities between servant leadership and transformational leadership, but they differ in the leader’s focus. Servant leaders focus on their followers’ development and performance, whereas transformational leaders focus on getting followers to identify with the organization and engage in support of organizational objectives.

The five dimensions of transformational leadership includes:

Dimensions of Transformational leadership
(adapted from the 2017 State of DevOps Report)

• Vision: Has a clear concept of where the organization is going and where it should be in five years.
• Intellectual stimulation: Challenges followers to think about problems in new ways.
• Inspirational communication: Communicates in a way that inspires and motivates, even in an uncertain or changing environment.
• Supportive leadership: Demonstrates care and consideration of followers’ needs and feelings.
• Personal recognition: Praises and acknowledges achievement of goals and improvements in work quality; personally compliments others when they do outstanding work.

Good leaders build great teams, great technology, and great organizations. Transformational leadership enables practices that correlate with high performance, and it helps team members communicate and collaborate in pursuit of organizational goals. Such kind of leadership provides the foundation for a culture in which continuous experimentation and learning is part of everybody’s daily work.

The 2017 State of DevOps Report also takes an in-depth look at transformational leadership.

Conclusion

Today it still happens here and there that we cannot fulfill all the wishes of the management. The difference from when I started is that now I know proven principles and approaches, that help me deal with these situations. We now know how to handle these exceptional cases in true agile leadership and quickly work together to achieve our goals.

If we consider an IT organization from the outside, the majority of organizations worldwide are busy continuously reinventing themselves. Some IT organizations find this easier than others, and some have to leave the field completely in the face of these rapid market changes and disappear entirely.

But why is this and how can we do our best to avoid being one of the latter?

In today’s world, it is necessary to be able to react fast and flexible, but also cost-saving and with high quality, to the wishes of the customers.

Charles Darwin, as I have already mentioned in one of my other articles, made what I consider to be a very accurate statement, which is as true today as it was then:

It is not the strongest of the species that survives, nor the most intelligent. It is the one most adaptable to change.

Charles Darwin, 1809

So we should not strive to be the strongest species, but to be able to adapt best. Applied to IT organizations, this means that we should not react reactively to market changes, but should prepare as well as possible for the change that is definitely coming and consider it as part of our DNA.

Manage the change

From classic change management, we should be aware that changes not only have an impact on the IT organization, but also on each individual person within this organization.

Elisabeth Küberl-Ross, an American-Swiss psychiatrist, has described in “The stages of grief” how we react to change and which stages we go through more or less intensively.

It doesn’t seem to matter whether these changes have occurred in the private or professional sphere. We must be mindful that each person has their own way of dealing with change, and we should always meet them where they are and pick them up at the stage they are at.

Organizational performance

Organizational performance measures the ability of an organization to achieve commercial and non-commercial goals. Academic research has validated this measure and found it to be highly correlated to measures of return on investment (ROI), and it is robust to economic cycles.

Combined, commercial and non-commercial goals include:

• Profitability
• Productivity
• Market share
• Number of customers
• Quantity of products or services
• Operating efficiency
• Customer satisfaction
• Quality of products or services provided
• Achieving organization or mission goals

Capabilities and practices

The DevOps Research and Assessment (DORA) team, has identified and validated a set of capabilities and practices that drive higher software delivery and organizational performance. The DORA’s State of DevOps research program includes data from over 31,000 professionals worldwide and has now been conducted regularly every year since 2014.

It is the longest-running, academically rigorous research survey of its kind and provides an independent look at the practices and capabilities that drive high performance in technology delivery and ultimately organizational outcomes. This research uses behavioral science to identify the most effective and efficient ways to develop and deliver software.

If we superimpose the organizational performance goals on the idea of being highly adaptable, we can figure out what a high-performing IT organization will look like. We can keep our goals in mind while still being adaptive to market changes.

The intrinsic motivation to influence the goals is obvious and instead of being blindsided by the external influences, we can equip ourselves well for it with management and technical practices.

Culture and work environment

• Climate for learning: An organization with a climate for learning views learning as an investment that is needed for growth, not as a necessary evil, undertaken only when required.
• Westrum organizational culture: This model of organizational culture was developed by sociologist Dr Ron Westrum. It classifies organizations as pathological, bureaucratic, or generative based on levels of cooperation, how problems are surfaced, the extent to which the organization is siloed, and how people react to failure and novelty.
• Culture of psychological safety: In teams with a culture of psychological safety, team members trust each other, are able to resolve conflict, take calculated and moderate risks, speak up, and are more creative.
• Job satisfaction: People feel supported by their employers, have the tools and resources to do their work, and feel their judgement is valued.
• Identity: Employees identify with the organization they work for. They say that the organization is a good place to work. They feel that the organization cares about them. And they’re willing to put in extra effort to help the organization succeed.

Lean product development

• Working in small batches: The extent to which teams break products and features into small batches that can be completed in less than a week and released frequently, including the use of MVPs (minimum viable products).
• Visibility of work in the value stream: Whether teams understand the workflow from business to customers, and whether they have visibility into this flow, including the status of products and features.
• Customer feedback: Whether organizations actively and regularly seek customer feedback and incorporate it into their product design.
• Team experimentation: Whether development teams have the authority to create and change specifications as part of the development process without requiring approval.

Software delivery and operational performance

• Continuous delivery
• Autonomy, trust and voice
• Cloud infrastructure
• Disaster recovery testing
• No functional outsourcing
• Clear and lightweight change process
• Lean product development
• Lean management practices

I will go into these topics in one of my next articles.

Additional reading

• DevOps Research and Assessment (DORA)
• DORA DevOps Quick Check
• Latest State of DevOps Report

In my last article about Why do we need DevOps? I already mentioned a framework that can be used to introduce DevOps in an organization - CALMS.

The CALMS framework was originally invented by Jez Humble, co-author of “The DevOps Handbook” and “Accelerate”. It is used as a means to assess whether an organization is ready to adopt DevOps processes or how an organization is progressing in its DevOps transition.

The acronym stands for Culture, Automation, Lean, Measurement and Sharing.

Although this is my preferred way to assess, the approaches “The Tree Ways”, described in “The Phoenix Project” or “The DevOps Handbook” and “Mature capabilities in technical and management practices” found in high-performing DevOps teams, based on the research presented in “Accelerate” are also mentioned.

Nevertheless, I would like to talk a bit more about the CALMS framework.

What DevOps is NOT

But before I do that, I would like to clarify what DevOps is NOT from my perspective by some debates in my nearer past. I have already mentioned some of these in passing in my last article. However, I would like to emphasize it clearly once again:

• DevOps is not the simple combination of Development and Operations teams: “We turn the development and operations team into a team - and voila we make DevOps.” If it were that simple, this article would lose its meaning.
• DevOps is not a tool: Even though harmonizing and finding the right toolstack may be important for the corresponding product development.
• DevOps is not a separate team: It may seem at the beginning of a transition as if a test balloon in a separately formed team would be easier. At the end of these balloon, this should no longer be an option for an organization and in the best case it has also shown that it can only work if DevOps is practiced in a team.
• DevOps is not a one-size-fits-all solution: There are so many business drivers and product development approaches. To suggest that the specific needs of a client can be met with one and the same solution is ridiculous.
• DevOps is not Cloud: I often hear DevOps in connection with cloud and the opinion that DevOps is the synonym for “Everything in the cloud”. Of course it’s not, there are a lot of businesses that are hard to do in the cloud, but DevOps works great there anyway.
• DevOps is not automation: Automation is often used interchangeably with DevOps. This is of course complete nonsense. A part of DevOps is also automation, as we will see in the following article. To find out what DevOps really is I recommend my article about Why do we need DevOps?

And there is one point which, for my personal conviction, must be mentioned. I say it frankly and I stand by it: “DevOps is not a role”

CALMS

Culture

Behind the aspect of Culture, which occurs in all three approaches, hides a complex problem which in many cases is also the main reason for the failure of a product development.

Before thinking about breaking down silos in an organization, a common view of the entire system and common targets should be in place and a foundation for a common responsibility should be laid. This is key if you want to be successful.

In most cases a common team vision and mission can be helpful and is one of the things that can make the difference. There are of course different approaches to developing these, but what is needed is trust, not only the trust of the team members themselves, but also the trust of management in the journey of DevOps and the ROI that comes with it.

To be able to destroy silos, cooperation must be encouraged. From the history the developers want to get new features into production as soon as possible. But in operation it’s more about being able to run the system as stable and sustainable as possible and not to endanger the stability by changes that have possibly hardly been tested. Thus the two statements from the history stand in contrast to each other. Which in some cases can lead to points of friction in communication. Which I will not go into here, I think you know the points of discussion.

That’s why it’s important to get everyone involved as early as possible and create a “You build it, you run it, you own it” mentality. The way a system is operated finally has an impact on the architecture and on the cooperation between components. If a system cannot be operated, it is ultimately doomed to failure.

Products where all parties involved have an open and encouraging communication culture are also the products that will be on the market in the coming years, other products have already disappeared due to lack of maintainability and many other aspects.

With the open culture comes transparency in communication. Successes as well as failures are part of this culture and are celebrated one way or another. Failures lead us to improve in what we do and to continuously work on ourselves, to learn something new and to improve our processes, our daily work.

Even though sharing successes and failures are actually more part of Sharing, they are deeply rooted in the culture. Teams that are not familiar with this culture can slowly be introduced to these aspects through games and a lot of motivation. Decisive for this is a non-blaming culture and the open discussion, the open ear of the team members and management.

Herewith I am approaching the end of the cultural aspect, even if there is still a lot to tell, to be fair. The last point I would like to mention is the topic of team structure or team constellation.

I have seen many companies that always seemed to be seeking the same stereotypes and thus inevitably created one-dimensional thinking. But to be successful, you need different types in a team. This does not start at seniority level (junior, professional, senior). It is an advantage to have different skill sets in one team. What I want to create are cross-functional teams that cover the complete software development life cycle and thus represent the non-plus-ultra for my system.

What it does not mean is that a team should only contain specialists and hard-to-manage individualists. It should mean that in order to develop successful products, it needs not only specialists in their field of expertise, but also generalists who can possibly cover one or more main areas of the SDLC.

Automation

To put it simply, DevOps teams should strive to automate as many manual and recurring tasks as possible. Attention should always be paid to stability, maintainability and simplicity. There are a few points that can be addressed under the main topic of automation. Each of the following principles separately fills whole articles and books, so I would have liked to mention them but will not go into all of them in detail:

• Automate the build process: Here we should name Continuous Integration as THE principle they should adopt. This includes the right branching strategy as well as automated versioning (e.g.: Semantic Versioning), but of course most of all the automated building of the solution to be able to use a build artifact for further steps.
• Automate the testing process: There are myths surrounding the testing process within DevOps. Is it a closed process within an SDLC? No. What is sufficient test coverage? Depends. Should we work on our unit tests and do we need integration tests at all? Definitely. Is the feature user friendly and client led? I don’t know. Are our test cases good and fast enough for our pipeline? Probably not. - What am I trying to say? Continuous Quality is a holistic concept that starts with the writing of user stories and ends with the involvement of the client in the process … or even goes beyond that. To make a product successful, you should always try to implement quality where you can. Is 20% unit test coverage sufficient? Probably not, if the product is used for more than 3 months. But does it need a coverage of around 100%? Probably not, if no lives depend on it. I could fill pages on quality, but I think we can leave it at one sentence: We need quality and quality assurance in our system so it doesn’t steal our sleep at the end of the day.
• Automate the infrastructure setup process: Infrastructure as Code (IaC) with the probably well known tools like Ansible, Chef, Terraform, Puppet, Kubernetes or similar, are part of it as well as approaches of GitOps.
• Automate the deployment or release process: Not only do principles such as Continuous Delivery or Deployment play a major role in this, but there are also issues that are perhaps not so obvious at first glance. These include the automation of a changelog, release notes, archiving artifacts for the necessary traceability and recovery in case of a pitfall.
• Automate the security process: It starts with static code analysis and ends with compliance policies. In the middle we deal with topics such as Dynamic Application Security Testing (DAST), Static Application Security Testing (SAST), Dependency scanning, Container scanning or Secret detection.
• Automate the monitoring process: Already at the beginning of the development of a product you should, as mentioned, think about how you want to operate it, but not only that. Besides monitoring the infrastructure, you also have to think about possible application-specific things. Log-file analysis or even more specifically, the correct writing of log output is essential. Also the feedback from clients or the collection of data from A/B or feature tests is an important part of monitoring that should be considered.

Automate the … boring stuff

Lean

Development teams use lean principles to eliminate waste by defining the client value and understand what value is, optimize the value stream, such as minimizing working in progress (WIP), making work and progress visible and traceable, reducing handover complexity and breaking down steps to ensure that the flow of the remaining steps run smoothly without interruptions and waiting times. This also includes the introduction of cross-functional teams and the training of employees who are versatile and adaptable.

Measurement

In order to better understand the possibilities of the current system it is helpful to get some things settled. In order to better assess the health of the product, I need well-considered metrics that can give us a starting point for improvement.

In this area we try to collect and analyse data. This can be planning data, product data, quality data or more general team data. It is important to mention that this is always about trust and we plead for the data to be used correctly in the right hands. We do not want to create a surveillance state, we want to get the best out of it for our clients and our product.

As mentioned, Continuous Learning and Improvement should be a key factor in our DevOps culture. A nice saying that I like to use myself is “Continuous Improvement is the only Constant in DevOps”

So what do we want to achieve:

• Collect & analyze product and system specific data
• Define metrics and thresholds
• Monitor and track metrics and automate notifications
• Identify mistakes
• Define quality gates
• Create a Continuous Learning and Improvement culture
• Improve efficiency and reduce cycle times

Sharing

Classically, this includes establishing a non-blaming culture, which sounds simple at first glance, but which, as I have learned, is apparently in our nature to reject fault. It requires a lot of experience and understanding and the management should lead by example.

An open communication culture should encourage the ask/share principle. We want to focus on solving difficulties and not on assigning blame. In this sense, it should also be emphasized that it is always about the overall system and not about micro-problems without looking at the other areas.

We can achieve this by focusing on Collective Code Ownership and by putting the team in focus. All in all - “Sharing is caring” and we should always keep this in mind. If we don’t care about the team and the product, why should our clients?

I am eager to hear your opinion and welcome any comments on this topic. So long - stay tuned and healthy!

In some of my projects, including my last one, I have to deal with inhomogeneous toolstacks.

Currently we work with tools like Atlassian Confluence, Jira Software, GitLab (CI), AWS and many more. Basically everything is no problem, but little by little the idea arose to combine the planning instrument, to support the agile methodology, with the development.

The team had problems in the traceability of changes in the source code, there was no or only a rudimentary connection between Jira and GitLab. The important aspect for the management to react to obstacles was missing. There was little feedback from development or the status of the product back to business.

In general I am a friend of documentation directly at the source code and so I often suggest Markdown as a basis - similar to my blog. And so I had the idea to at least put the product specific documentation into GitLab. This thought spread further and the idea to use GitLab completely was born.

GitLab (CI) is one of the best tools available today for continuous integration and continuous delivery/deployment, but is that true for Agile Software Development? Can it stand up to a giant like Jira Software? If I compare the Agile artifacts (Epic, User Story, Task, Points and Estimation, Product backlog, Sprint/iteration, Charts, Agile board), GitLab has a solution for all artifacts. So the answer for me is YES - but a long-term study is still pending.

We made the decision to switch from Jira to GitLab for our product. But how do I get the artifacts collected over time in Jira to it?

Well, GitLab offers a rudimentary import for Jira Ticket starting with version 12.10. The description seems simple, the connection between Jira and GitLab was quickly established, and the import was started with just over 2000 tickets …

What can I say, it seems that GitLab still has some problems with the import. That’s why the import was never finished and there’s no way to restart it or stop the old import. According to GitLab support, the feature is fairly new and an MVP. If you’re looking for more, you’ll find plans on where the feature should go.

• https://gitlab.com/gitlab-org/gitlab/-/issues/2780
• https://gitlab.com/gitlab-org/gitlab/-/issues/217395
• https://gitlab.com/gitlab-org/gitlab/-/issues/214812
• https://gitlab.com/gitlab-org/gitlab/-/issues/214810
• https://gitlab.com/gitlab-org/gitlab/-/issues/210580

But what can I do in this situation? Of course, I could hope that the ticket will be fixed relatively quickly and we can continue with the import in a timely manner. But since I still have some developer gene in me, I have decided to write my own importer.

Jira 2 GitLab

Jira and GitLab each offer good RESTful APIs that don’t require much. For more details about the preconditions I refer you directly to the corresponding developer documentation.

Jira:

• https://developer.atlassian.com/cloud/jira/software/rest/

GitLab:

• https://docs.gitlab.com/ee/api/
• https://docs.gitlab.com/ee/api/api_resources.html

The following scripts were tested with Jira Cloud and GitLab.com (Cloud) 12.10. For the scripts I used Python3 and locally Python v3.7.3.

Epics

In GitLab, epics are managed on a group level and should be viewed separately. Epics are used at this level for roadmap planning and can be planned across multiple projects.

NOTE Epics are only available with a GitLab.com Gold Subscription.

# import_epics.py
import requests 
from requests.auth import HTTPBasicAuth
import json

## Jira specifics
# Jira URL
JIRA_URL = 'https://your-jira-instance.com/'
# Jira user credentials (incl. API token)
JIRA_ACCOUNT = ('your-jira-username', 'your-jira-api-token')
# Jira project ID (short)
JIRA_PROJECT = 'PRJ'
# Jira Query (JQL)
JQL = 'project=%s+AND+issueType=Epic+AND+resolution=Unresolved+ORDER+BY+createdDate+ASC&maxResults=100' % JIRA_PROJECT

# *False* if Jira / GitLab is using self-signed certificates, otherwhise *True*
VERIFY_SSL_CERTIFICATE = True

# Read Jira Epics
response = requests.get(
  JIRA_URL + 'rest/api/latest/search?jql=' + JQL,
  auth=HTTPBasicAuth(*JIRA_ACCOUNT),
  verify=VERIFY_SSL_CERTIFICATE,
  headers={'Content-Type': 'application/json'}
)

if response.status_code != 200:
  raise Exception("Unable to read Epics from %s!" % JIRA_PROJECT)

jira_issues = response.json()

for issue in jira_issues['issues']:
  print("Import Epic with Jira-Key " + issue['key'])

Once you run the script, you should see a similar output:

$ Import Epic with Jira-Key PRJ-123
$ Import Epic with Jira-Key PRJ-456

Now we can start with the transfer of the Jira Epics. For this we use the GitLab API. We need a few more variables for that. The easiest way is to take over title and description from Jira. But for a more complex variant you could also add labels, start and due date or more. See the GitLab API.

## GitLab specifics
GITLAB_URL = 'https://gitlab.com/'
# GitLab token will be used whenever the API is invoked
GITLAB_TOKEN = 'your-private-gitlab-token'
# GitLab group that you are importing to
GITLAB_GROUP = 'your-group-name'
# GitLab group id.
GITLAB_GROUP_ID = 'your-group-id'

...

for issue in jira_issues['issues']:
  print("Import Epic with Jira-Key " + issue['key'])

  title = issue['fields']['summary']
  description = issue['fields']['description']

  data = { 
    'title': title,
    'description': description
  }

  response = requests.post(
    GITLAB_URL + 'api/v4/groups/%s/epics' % GITLAB_GROUP_ID,
    headers={'PRIVATE-TOKEN': GITLAB_TOKEN},
    verify=VERIFY_SSL_CERTIFICATE,
    data=data
  )

NOTE One more point should be mentioned in addition. The Jira API restricts the data sets to a maximum of 100. This has to be solved by pagination if you have more than 100 epics.

You can find the complete source code here.

Issues

If we take the script for the Epics as a basis, we can do the same with other ticket types like Stories, Sub-tasks, Tasks, or Spikes. For this I would like to share some code snippets that might be useful for your import.

Jira

# Get Jira attachments and comments
requests.get(
  JIRA_URL + 'rest/api/latest/issue/%s/?fields=attachment,comment' % issue['id'],
  auth=HTTPBasicAuth(*JIRA_ACCOUNT),
  verify=VERIFY_SSL_CERTIFICATE,
  headers={'Content-Type': 'application/json'}
)

GitLab

# Get GitLab milestones
requests.get(
  GITLAB_URL + 'api/v4/projects/%s/milestones' % GITLAB_PROJECT_ID,
  headers={'PRIVATE-TOKEN': GITLAB_TOKEN},
  verify=VERIFY_SSL_CERTIFICATE
)

# Get GitLab users
requests.get(
  GITLAB_URL + 'api/v4/users',
  headers={'PRIVATE-TOKEN': GITLAB_TOKEN},
  verify=VERIFY_SSL_CERTIFICATE
)

# Create new GitLab issue

data = { ... }

repsonse = requests.post(
  GITLAB_URL + 'api/v4/projects/%s/issues' % GITLAB_PROJECT_ID,
  headers={'PRIVATE-TOKEN': GITLAB_TOKEN},
  verify=VERIFY_SSL_CERTIFICATE,
  data=data
)

gl_issue = response.json()

# Close GitLab issues that were already closed in Jira
if issue['fields']['status']['statusCategory']['key'] == "done":
  requests.put(
    GITLAB_URL + 'api/v4/projects/%s/issues/%s' % (GITLAB_PROJECT_ID, gl_issue['iid']),
    headers={'PRIVATE-TOKEN': GITLAB_TOKEN},
    verify=VERIFY_SSL_CERTIFICATE,
    data={'state_event': 'close'}
  )

See Gist for the whole source code.

Remove unused GitLab labels

I also created a script to remove unused labels in GitLab. You can find it here.

Additional Notes

NOTE With GitLab.com you currently do not have the ability to execute SUDO commands. Therefore no user mapping can be done with it. All data is done with the user (private-token) who is also importing the data.

A long time ago, it was at the beginning of the 18th century, more precisely in 1809, when Charles Darwin made a statement that was formative for that time, a statement that is still valid today and will probably survive for a long time.

It is not the strongest of the species that survives, nor the most intelligent. It is the one most adaptable to change.

Charles Darwin, 1809

The statement itself has been adapted somewhat over time, but its basic message has not changed and in times of COVID-19 became more relevant than ever before.

It lays the foundation for something else, it’s part of an area in DevOps that is often simply called a mindset.

Of course, one could now claim that the mindset is not the only thing that is important in the software development life cycle (SDLC). There are things that are just as important as mindset like principles, practices and it is also important to think about harmonizing tools. Nevertheless, without the right mindset, all of the above mentioned are only short-lived painkillers.

Nevertheless, let me briefly explain why we need DevOps with all its aspects so desperately. I want to start with a look at anti-patterns, because it is always easier to find things that DON’T work. And don’t worry, I will come up with solutions later.

Why IT projects fail

Gartner studies suggest that 75% of all IT projects in the USA are seen as failures by those who initiated them.

The total number of failed projects results from different areas. Project Initiation and Planning Issues, Technical and Requirements Issues, Project Management Issues and Stakeholder Management and Team Issues.

Many problems can be found especially in the latter area “Stakeholder Management and Team Issues”.

Stakeholder Management and Team Issues

If we now look at some points why this area is so problematic, the cry for a fix becomes obvious:

• Insufficient attention to stakeholders and their needs
• Failure to manage expectations
• Lack of senior management/executive support
• Inadequate visibility of project status
• Denial adopted in preference to hard truths
• People not dedicated to project
• Project team members lack experience and do not have the required skills
• Team lacks authority or decision making ability
• Poor collaboration, communication and teamwork

It is the same old thinking, that leads to the same old result

I don’t want to claim that the above statement is mine, but I think that this describes one of the biggest problems in today’s SDLC – the inability to adapt to change.

What I want to convey is that it is often not the tools or the technology that make a project fail. But there are other challenges in the SDLC that we will now look at.

The Challenges

As an IT consultant and DevOps advocate I have seen a lot of different customer scenarios in the past years and even though, I see different roles in different projects, I also see a lot of similarities. I would like to share those similarities with you and maybe you feel familiar with them:

Now that we know some of the challenges in an SDLC, we should take a look at the idea of DevOps and how it can be a cure to the itch.

The Idea

If you search the Internet for an explanation of what DevOps exactly means, you will find many different views. Tool vendors try to make it a tool problem, consultants try to make it a fluffy mindset thing. I think the solution lies somewhere in the middle and I have therefore come up with my very own explanation, which I have compiled for you below.

DevOps is organizational culture and leadership with lean and agile principles combined with technical practices to enable continuous delivery of value to end users.

These drive organizational performance and technology performance.

You could say that this is a high-level explanation, but it helped me a lot to quickly get into conversation with the C-level management.

The reason for that is that this summary covers most of the challenges mentioned before, of course on a very abstract level but also in a very compact and meaningful way and that is always a good way to start a conversation.

What’s so special about DevOps?

DevOps is based on agility and the agile development. But it goes at least one step further than that, it includes the complete business side and operations. Using the principles of automation, for example, it attempts to gradually level out the overheads that have been built up or turns them into something that adds value.

What DevOps can thus accomplish is:

• Higher quality
• Faster delivery
• Lower costs
• More flexibility

So if DevOps manages to make only half of the above mentioned IT projects a success, then we should definitely invest in DevOps transitions and start sooner rather than later.

However, it should also be clearly mentioned that DevOps does not make sense for every project and every organization, but a lot of its principles and practices are always applicable and bring a quick ROI for almost every project and every organization.

How can DevOps be applied to real life scenarios and how can we take the idea of DevOps and turn the mentioned challenges into opportunities?

An approach that I personally found very useful in many situations is the conceptual framework CALMS. Thereby the acronym stands for Culture, Automation, Lean, Measurement and Sharing. You will read about it soon in an upcoming blog of mine.

Until then I am very interested in your thoughts, experiences and feedback. Stay tuned and healthy!

If we think about a migration of databases, the analysis will probably not get past Flyway.

Flyway has focused on exactly one task and in this area I think they are doing a very good job.

The Community Edition comes with a good set of features. For the development in the Postgres area this seems to be sufficient, especially in the beginning. It is important to note the “Guaranteed database support timeline”. The Community Edition comes with a 5-year guarantee. Currently this would mean the following for Postgres.

Supported versions: 12, 11, 10, 9.6, 9.5, 9.4 The Enterprise Edition would extend support to these versions: 9.3, 9.2, 9.1, 9.0

As soon as you think about Oracle PL*SQL and want to use it in your project, there is no way around a Pro or Enterprise Edition.

As soon as you have decided on a version, it is time to prepare your first “migration”.

Of course you can download Flyway with its cross-platform capability to your system and use the CLI of it.

However, I would suggest to use a dockerized version right away. The advantage is that you can reuse this approach later in a Continuous Integration / Delivery Pipeline.

The info CLI command prints the details and status information about all the migrations.

flyway/flyway info

Flyway also supports different JDBC drivers, if these should not be sufficient, e.g. with Oracle (ojdbc8), then one can simply reinstall the drivers or tell Flyway where the drivers can be found.

A list of useful CLI commands can be found here.

Flyway can be used quite nicely as a build image as well and it is almost as easy as using the CLI directly.

$ docker pull flyway/flyway:6-alpine
$ docker run --rm -v ${pwd}/sql:/flyway/sql flyway/flyway:6-alpine -url=jdbc:postgresql://<HOST>/<DBNAME> -user=<USER> -password=<PASSWORD> info

Instead of downloading the Flyway executable file manually, you can download the Docker image by ‘docker pull’ as shown above. Of course you have to tell Flyway where to find the scripts, which you can easily do with volumes. It is also necessary to establish a connection to the database. After that the respective CLI command can be used.

If we get along with the standard JDBC drivers, we can stop here. However, if other drivers are needed, I recommend to create your own Docker image with the appropriate driver pre-installed and possibly even a license defined.

How this works can be seen afterwards:

## Base: https://github.com/flyway/flyway-docker
FROM flyway/flyway:latest-alpine

LABEL maintainer="juergenpointinger" \
      description="This is a build container image to interact with Flyway"

## Supported Volumes
# /flyway/conf: Directory containing a flyway.conf configuration file
# /flyway/drivers: Directory containing the JDBC driver for your database
# /flyway/sql: The SQL files that you want Flyway to use (for SQL-based migrations)
# /flyway/jars: The jars files that you want Flyway to use (for Java-based migrations)
VOLUME [ "/flyway/conf","/flyway/drivers", "flyway/sql", "flyway/jars" ]

# Use specific Oracle JDBC driver
COPY ojdbc8.jar "/flyway/drivers"

## Flyway Edition
# community: Select the Flyway Community Edition (default)
# pro: Select the Flyway Pro Edition
# enterprise: Select the Flyway Enterprise Edition
ENV FLYWAY_EDITION=community

Afterwards the newly created image can be used just like the original Flyway image. Have fun with Flyway.

In most projects it is recommended to use a scoped (private) registry to share self-developed modules.

Sometimes it is not possible or makes no sense to use tools like JFrog Artifactory or Sonatype Nexus Repository.

In GitLab, such registries can now be used relatively easily.

Add the GitLab NPM Registry to your local or global NPM configuration. Replace @your-scope with your specific scope name (e.g. your organization name):

$ npm config set @your-scope:registry https://gitlab.com/api/v4/packages/npm/

Your config output look like this (on Windows):

; cli configs
metrics-registry = "https://registry.npmjs.org/"
...

; userconfig C:\Users\<userprofile>\.npmrc
@your-scope:registry = "https://gitlab.com/api/v4/packages/npm/"

Now you just need to authenticate with the newly created scoped registry. Replace <your_token> with your personal access token:

$ npm config set '//gitlab.com/api/v4/packages/npm/:_authToken' "<your_token>"

The commands described above change your NPM userconfig, your user specific .npmrc file. If you want to make the change on a global level, you would have to add “–global” at the end.

$ npm config set @your-scope:registry https://gitlab.com/api/v4/packages/npm/ --global
$ npm config set '//gitlab.com/api/v4/packages/npm/:_authToken' "<your_token>" --global

The .npmrc in a global context:

; cli configs
metrics-registry = "https://registry.npmjs.org/"
...

; globalconfig C:\Program Files\nodejs\etc\npmrc
@your-scope:registry = "https://gitlab.com/api/v4/packages/npm/"

To make it easier for me, I started some time ago to create a cheat sheet for Docker. Since I don’t want to keep this knowledge only for myself, I share some of the most important CLI commands I need in my daily do-ing.

Login

When you start to deal with continuous integration tools, you quickly get to the point where you also use a private container registry.

Docker provides a very simple way to use a registry and log in with your credentials:

$ docker login registry.gitlab.com

To make the docker login command non-interactive, such as required for GitLab CI, you can set the --password-stdin flag to provide a password via STDIN.

Using STDIN prevents the password from ending up in the shell’s history or log files.

$ echo "${CI_REGISTRY_PASSWORD}" | docker login -u "${CI_REGISTRY_USER}" "${CI_REGISTRY}" --password-stdin

System health

In order to check the current Docker Disk Usage and thus the allocation of the system a simple command

$ docker system df

Purging all unused or dangling Images, Containers, Volumes, and Networks

Docker provides a single command that will clean up any resources — images, containers, volumes, and networks — that are dangling (not associated with a container):

$ docker system prune

To additionally remove any stopped containers and all unused images (not just dangling images), add the -a flag to the command:

$ docker system prune -a

Removing Docker Images

Removing dangling (untagged) Images

Docker images consist of multiple layers. Dangling images are layers that have no relationship to any tagged images. They no longer serve a purpose and consume disk space. They can be located by adding the filter flag, -f with a value of dangling=true to the docker images command. When you’re sure you want to delete them, you can use one of the following commands:

$ docker rmi $(docker images -f "dangling=true" -q)

or

$ docker image prune

NOTE: If you build an image without tagging it, the image will appear on the list of dangling images because it has no association with a tagged image.

Removing specific Images by Pattern

Sometimes it is necessary to search an image for a certain <<pattern>>. A combination of Docker CLI and grep can help.

$ docker images -a |  grep "<<pattern>>"

If you are comfortable with the search/filter, you can use ‘awk’ to pass the IDs to the Docker CLI:

$ docker image rm $(docker images -a | grep "<<pattern>>" | awk '{print $3}')

or

$ docker images -a | grep "<<pattern>>" | awk '{print $3}' | xargs docker image rm

NOTE: Instead of image rm you could use the shorter version with rmi

Removing all Images

Also the deletion of all images can be solved with the Docker CLI. To do this, you can simply append -a to the command. -q returns the Image ID, which can be used to pass it on to other commands.

$ docker image rm $(docker images -a -q)

or

$ docker image rm $(docker image ls -q)

Removing Docker Containers

Similar to the docker images command, you can also add the parameter -a to the docker ps command, which lists the running containers, so that all containers are shown, even those that are no longer running.

The displayed container ID or name can then be reused:

$ docker container rm <<container_id/name>>

It is also possible to display only the exited containers:

$ docker ps -a -f status=exited

Removing specific Containers by Pattern

And similar to the images it works with the containers.

$ docker container rm $(docker ps -a | grep "<<pattern>>" | awk '{print $1}')

or

$ docker ps -a | grep "<<pattern>>" | awk '{print $1}' | xargs docker container rm

NOTE: In this case {print $1} returns the ID of the container.

Removing all stopped Containers

With the knowledge we have just gained, we can now simply delete all the containers that have exited.

# Historical command
$ docker rm -f $(docker ps -aq -f status=exited)

or

$ docker container prune

NOTE: We again only pass the ID to the Docker CLI using -q.

Removing all Containers

If we need to remove all containers, the running ones and the stopped ones we can use the following command.

$ docker container rm -f $(docker container ls -aq)

Removing all Volumes

To remove the volumes not used any longer, we can use.

$ docker volume rm $(docker volume ls -q)

or

$ docker volume prune

Removing all Builder cache

To remove the build cache, we can use.

$ docker builder prune

Stats for all running Containers

The docker stats command is used to display a live stream of container resource usage statistics. This command can be extended to display the statistics of all containers simultaneously.

$ docker ps -q | xargs docker stats