1004 lines
46 KiB
HTML
1004 lines
46 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en-us">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
|
||
|
||
<script>
|
||
(function() {
|
||
|
||
|
||
const autoTheme = false;
|
||
if (autoTheme) {
|
||
document.documentElement.setAttribute('data-auto-theme', 'true');
|
||
}
|
||
|
||
const theme = localStorage.getItem('cleanwhite-theme') ||
|
||
(window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light');
|
||
document.documentElement.setAttribute('data-theme', theme);
|
||
})();
|
||
</script>
|
||
|
||
|
||
<meta property="og:site_name" content="David Young">
|
||
<meta property="og:type" content="article">
|
||
|
||
|
||
<meta property="og:image" content="https://davidpaulyoung.com//img/2018-04-16-using-helm-to-deploy-to-kubernetes/buffalo.jpg">
|
||
<meta property="twitter:image" content="https://davidpaulyoung.com//img/2018-04-16-using-helm-to-deploy-to-kubernetes/buffalo.jpg" />
|
||
|
||
|
||
|
||
<meta name="title" content="Helm介绍" />
|
||
<meta property="og:title" content="Helm介绍" />
|
||
<meta property="twitter:title" content="Helm介绍" />
|
||
|
||
|
||
|
||
<meta name="description" content="Helm是Kubernetes生态系统中的一个软件包管理工具。本文将介绍为何要使用Helm进行Kubernetes软件包管理，澄清Helm中使用到的相关概念，并通过一个具体的示例学习如何使用Helm打包，分发，安装，升级及回退Kubernetes应用。">
|
||
<meta property="og:description" content="Helm是Kubernetes生态系统中的一个软件包管理工具。本文将介绍为何要使用Helm进行Kubernetes软件包管理，澄清Helm中使用到的相关概念，并通过一个具体的示例学习如何使用Helm打包，分发，安装，升级及回退Kubernetes应用。" />
|
||
<meta property="twitter:description" content="Helm是Kubernetes生态系统中的一个软件包管理工具。本文将介绍为何要使用Helm进行Kubernetes软件包管理，澄清Helm中使用到的相关概念，并通过一个具体的示例学习如何使用Helm打包，分发，安装，升级及回退Kubernetes应用。" />
|
||
|
||
|
||
<meta property="og:url" content="https://davidpaulyoung.com/2018/04/16/using-helm-to-deploy-to-kubernetes/" />
|
||
|
||
|
||
<meta property="twitter:card" content="summary" />
|
||
|
||
|
||
|
||
<meta name="keyword" content="Von Balthasar, Scripture, Gravel Riding, Ham Radio, Divine Office, Open Source">
|
||
<link rel="shortcut icon" href="/img/favicon.ico">
|
||
|
||
<title>Helm介绍 | David Young Blog</title>
|
||
|
||
<link rel="canonical" href="/2018/04/16/using-helm-to-deploy-to-kubernetes/">
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" href="/css/bootstrap.min.css">
|
||
|
||
|
||
<link rel="stylesheet" href="/css/hugo-theme-cleanwhite.css">
|
||
|
||
|
||
<link rel="stylesheet" href="/css/theme-variables.css">
|
||
|
||
|
||
<link rel="stylesheet" href="/css/zanshang.min.css">
|
||
|
||
|
||
<link rel="stylesheet" href="/css/font-awesome.all.min.css">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script src="/js/jquery.min.js"></script>
|
||
|
||
|
||
<script src="/js/bootstrap.min.js"></script>
|
||
|
||
|
||
<script src="/js/hux-blog.min.js"></script>
|
||
|
||
|
||
<script src="/js/lazysizes.min.js"></script>
|
||
|
||
|
||
|
||
|
||
</head>
|
||
|
||
|
||
|
||
|
||
|
||
<nav class="navbar navbar-default navbar-custom navbar-fixed-top">
|
||
|
||
<div class="container-fluid">
|
||
|
||
<div class="navbar-header page-scroll">
|
||
<button type="button" class="navbar-toggle">
|
||
<span class="sr-only">Toggle navigation</span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
<a class="navbar-brand" href="/">David Young</a>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="huxblog_navbar">
|
||
<div class="navbar-collapse">
|
||
<ul class="nav navbar-nav navbar-right">
|
||
<li>
|
||
<a href="/">All Posts</a>
|
||
</li>
|
||
|
||
|
||
<li>
|
||
<a href="/categories/life/">life</a>
|
||
</li>
|
||
|
||
<li>
|
||
<a href="/categories/tech/">tech</a>
|
||
</li>
|
||
|
||
<li>
|
||
<a href="/categories/tips/">tips</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li><a href="/archive//">ARCHIVE</a></li>
|
||
|
||
<li><a href="/notes//">NOTES</a></li>
|
||
|
||
<li><a href="/about//">ABOUT</a></li>
|
||
|
||
<li>
|
||
<a href="/search"><i class="fa fa-search"></i></a>
|
||
</li>
|
||
|
||
<li>
|
||
<a href="#" id="theme-toggle" title="Toggle dark mode" style="opacity: 0;">
|
||
<i class="fa fa-moon"></i>
|
||
<i class="fa fa-sun" style="display: none;"></i>
|
||
</a>
|
||
</li>
|
||
<script>
|
||
(function() {
|
||
|
||
var theme = localStorage.getItem('cleanwhite-theme') ||
|
||
(window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light');
|
||
var toggleBtn = document.getElementById('theme-toggle');
|
||
if (toggleBtn) {
|
||
var moonIcon = toggleBtn.querySelector('.fa-moon');
|
||
var sunIcon = toggleBtn.querySelector('.fa-sun');
|
||
|
||
if (theme === 'dark') {
|
||
if (moonIcon) moonIcon.style.display = 'none';
|
||
if (sunIcon) sunIcon.style.display = 'inline';
|
||
toggleBtn.setAttribute('title', 'Switch to light mode');
|
||
} else {
|
||
if (moonIcon) moonIcon.style.display = 'inline';
|
||
if (sunIcon) sunIcon.style.display = 'none';
|
||
toggleBtn.setAttribute('title', 'Switch to dark mode');
|
||
}
|
||
|
||
|
||
requestAnimationFrame(function() {
|
||
toggleBtn.style.transition = 'opacity 0.2s ease';
|
||
toggleBtn.style.opacity = '1';
|
||
});
|
||
}
|
||
})();
|
||
</script>
|
||
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</nav>
|
||
<script>
|
||
|
||
|
||
|
||
var $body = document.body;
|
||
var $toggle = document.querySelector('.navbar-toggle');
|
||
var $navbar = document.querySelector('#huxblog_navbar');
|
||
var $collapse = document.querySelector('.navbar-collapse');
|
||
|
||
$toggle.addEventListener('click', handleMagic)
|
||
function handleMagic(e){
|
||
if ($navbar.className.indexOf('in') > 0) {
|
||
|
||
$navbar.className = " ";
|
||
|
||
setTimeout(function(){
|
||
|
||
if($navbar.className.indexOf('in') < 0) {
|
||
$collapse.style.height = "0px"
|
||
}
|
||
},400)
|
||
}else{
|
||
|
||
$collapse.style.height = "auto"
|
||
$navbar.className += " in";
|
||
}
|
||
}
|
||
|
||
|
||
document.addEventListener('DOMContentLoaded', function() {
|
||
var navLinks = document.querySelectorAll('.navbar-collapse a');
|
||
navLinks.forEach(function(link) {
|
||
link.addEventListener('click', function() {
|
||
if ($navbar.className.indexOf('in') > 0) {
|
||
|
||
$navbar.className = " ";
|
||
setTimeout(function(){
|
||
if($navbar.className.indexOf('in') < 0) {
|
||
$collapse.style.height = "0px"
|
||
}
|
||
},400)
|
||
}
|
||
});
|
||
});
|
||
});
|
||
</script>
|
||
|
||
|
||
|
||
|
||
<style type="text/css">
|
||
header.intro-header {
|
||
background-image: url('/img/2018-04-16-using-helm-to-deploy-to-kubernetes/buffalo.jpg')
|
||
}
|
||
</style>
|
||
|
||
<header class="intro-header" >
|
||
|
||
<div class="container">
|
||
<div class="row">
|
||
<div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
|
||
<div class="post-heading">
|
||
<div class="tags">
|
||
|
||
<a class="tag" href="/tags/kubernetes" title="Kubernetes">
|
||
Kubernetes
|
||
</a>
|
||
|
||
<a class="tag" href="/tags/helm" title="Helm">
|
||
Helm
|
||
</a>
|
||
|
||
</div>
|
||
<h1>Helm介绍</h1>
|
||
<h2 class="subheading">强大的Kubernetes包管理工具</h2>
|
||
<span class="meta">
|
||
|
||
Posted by
|
||
|
||
赵化冰
|
||
|
||
on
|
||
Monday, April 16, 2018
|
||
|
||
|
||
|
||
|
||
</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</header>
|
||
|
||
|
||
|
||
|
||
<article>
|
||
<div class="container">
|
||
<div class="row">
|
||
|
||
|
||
<div class="
|
||
col-lg-8 col-lg-offset-2
|
||
col-md-10 col-md-offset-1
|
||
post-container">
|
||
|
||
|
||
<h2 id="前言">前言</h2>
|
||
<hr>
|
||
<p>Helm是Kubernetes生态系统中的一个软件包管理工具。本文将介绍为何要使用Helm进行Kubernetes软件包管理，澄清Helm中使用到的相关概念，并通过一个具体的示例学习如何使用Helm打包，分发，安装，升级及回退Kubernetes应用。</p>
|
||
<h2 id="kubernetes应用部署的挑战">Kubernetes应用部署的挑战</h2>
|
||
<hr>
|
||
<p>让我们首先来看看Kubernetes，kubernetes提供了基于容器的应用集群管理，为容器化应用提供了部署运行、资源调度、服务发现和动态伸缩等一系列完整功能。</p>
|
||
<p>kubernetes的核心设计理念是: 用户定义应用程序的规格，而kubernetes则负责按照定义的规则部署并运行应用程序，如果应用系统出现问题导致偏离了定义的规格，kubernetes负责对其进行自动修正。例如应用规格要求部署两个实例，其中一个实例异常终止了，kubernetes会检查到并重新启动一个新的实例。</p>
|
||
<p>用户通过使用kubernetes API对象来描述应用程序规格，包括Pod，Service，Volume，Namespace，ReplicaSet，Deployment，Job等等。一般这些对象需要写入一系列的yaml文件中，然后通过kubernetes命令行工具kubectl进行部署。</p>
|
||
<p>以下面的wordpress应用程序为例，涉及到多个kubernetes API对象，这些kubernetes API对象分散在多个yaml文件中。</p>
|
||
<p>图1： Wordpress应用程序中涉及到的kubernetes API对象
|
||
|
||
<img src="/img/2018-04-16-using-helm-to-deploy-to-kubernetes/wordpress.png" alt="">
|
||
|
||
</p>
|
||
<p>可以看到，在进行kubernetes软件部署时，我们面临下述问题：</p>
|
||
<ul>
|
||
<li>如何管理，编辑和更新这些这些分散的kubernetes应用配置文件？</li>
|
||
<li>如何把一套的相关配置文件作为一个应用进行管理？</li>
|
||
<li>如何分发和重用kubernetes的应用配置？</li>
|
||
</ul>
|
||
<p>Helm的引入很好地解决上面这些问题。</p>
|
||
<h2 id="helm是什么">Helm是什么？</h2>
|
||
<hr>
|
||
<p>很多人都使用过Ubuntu下的ap-get或者CentOS下的yum, 这两者都是Linux系统下的包管理工具。采用apt-get/yum,应用开发者可以管理应用包之间的依赖关系，发布应用；用户则可以以简单的方式查找、安装、升级、卸载应用程序。</p>
|
||
<p>我们可以将Helm看作Kubernetes下的apt-get/yum。Helm是Deis (<a href="https://deis.com/">https://deis.com/</a>) 开发的一个用于kubernetes的包管理器。</p>
|
||
<p>对于应用发布者而言，可以通过Helm打包应用，管理应用依赖关系，管理应用版本并发布应用到软件仓库。</p>
|
||
<p>对于使用者而言，使用Helm后不用需要了解Kubernetes的Yaml语法并编写应用部署文件，可以通过Helm下载并在kubernetes上安装需要的应用。</p>
|
||
<p>除此以外，Helm还提供了kubernetes上的软件部署，删除，升级，回滚应用的强大功能。</p>
|
||
<h2 id="helm组件及相关术语">Helm组件及相关术语</h2>
|
||
<hr>
|
||
<p>开始接触Helm时遇到的一个常见问题就是Helm中的一些概念和术语非常让人迷惑，我开始学习Helm就遇到这个问题。</p>
|
||
<p>因此我们先了解一下Helm的这些相关概念和术语。</p>
|
||
<ul>
|
||
<li>
|
||
<p>Helm</p>
|
||
<p>Kubernetes的应用打包工具，也是命令行工具的名称。</p>
|
||
</li>
|
||
<li>
|
||
<p>Tiller</p>
|
||
<p>Helm的服务端，部署在Kubernetes集群中，用于处理Helm的相关命令。</p>
|
||
</li>
|
||
<li>
|
||
<p>Chart</p>
|
||
<p>Helm的打包格式，内部包含了一组相关的kubernetes资源。</p>
|
||
</li>
|
||
<li>
|
||
<p>Repoistory</p>
|
||
<p>Helm的软件仓库，repository本质上是一个web服务器，该服务器保存了chart软件包以供下载，并有提供一个该repository的chart包的清单文件以供查询。在使用时，Helm可以对接多个不同的Repository。</p>
|
||
</li>
|
||
<li>
|
||
<p>Release</p>
|
||
<p>使用Helm install命令在Kubernetes集群中安装的Chart称为Release。</p>
|
||
</li>
|
||
</ul>
|
||
<blockquote>
|
||
<p>需要特别注意的是， Helm中提到的Release和我们通常概念中的版本有所不同，这里的Release可以理解为Helm使用Chart包部署的一个应用实例。</p>
|
||
<p>其实Helm中的Release叫做Deployment更合适。估计因为Deployment这个概念已经被Kubernetes使用了，因此Helm才采用了Release这个术语。</p>
|
||
</blockquote>
|
||
<p>下面这张图描述了Helm的几个关键组件Helm（客户端），Tiller（服务器），Repository（Chart软件仓库），Chart（软件包）之前的关系。</p>
|
||
<p>图2： Helm软件架构
|
||
|
||
<img src="/img/2018-04-16-using-helm-to-deploy-to-kubernetes/helm-architecture.png" alt="">
|
||
|
||
</p>
|
||
<h2 id="安装helm">安装Helm</h2>
|
||
<hr>
|
||
<p>下面我们通过一个完整的示例来介绍Helm的相关概念，并学习如何使用Helm打包，分发，安装，升级及回退kubernetes应用。</p>
|
||
<p>可以参考Helm的帮助文档https://docs.helm.sh/using_helm/#installing-helm 安装Helm</p>
|
||
<p>采用二进制的方式安装Helm</p>
|
||
<ol>
|
||
<li>下载 Helm <a href="https://github.com/kubernetes/helm/releases">https://github.com/kubernetes/helm/releases</a></li>
|
||
<li>解压 tar -zxvf helm-v2.0.0-linux-amd64.tgz</li>
|
||
<li>拷贝到bin目录 mv linux-amd64/helm /usr/local/bin/helm</li>
|
||
</ol>
|
||
<p>然后使用下面的命令安装服务器端组件Tiller</p>
|
||
<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#282a36;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-text-size-adjust:none;"><code class="language-bash" data-lang="bash"><span style="display:flex;"><span>Helm init
|
||
</span></span></code></pre></div><h2 id="构建一个helm-chart">构建一个Helm chart</h2>
|
||
<hr>
|
||
<p>让我们在实践中来了解Helm。这里将使用一个Go测试小程序，让我们先为这个小程序创建一个Helm chart。</p>
|
||
<pre tabindex="0"><code>git clone https://github.com/zhaohuabing/testapi.git;
|
||
cd testapi
|
||
</code></pre><p>首先创建一个chart的骨架</p>
|
||
<pre tabindex="0"><code>helm create testapi-chart
|
||
</code></pre><p>该命令创建一个testapi-chart目录，该目录结构如下所示，我们主要关注目录中的这三个文件即可: Chart.yaml，values.yaml 和 NOTES.txt。</p>
|
||
<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#282a36;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-text-size-adjust:none;"><code class="language-Bash" data-lang="Bash"><span style="display:flex;"><span>testapi-chart
|
||
</span></span><span style="display:flex;"><span>├── charts
|
||
</span></span><span style="display:flex;"><span>├── Chart.yaml
|
||
</span></span><span style="display:flex;"><span>├── templates
|
||
</span></span><span style="display:flex;"><span>│ ├── deployment.yaml
|
||
</span></span><span style="display:flex;"><span>│ ├── _helpers.tpl
|
||
</span></span><span style="display:flex;"><span>│ ├── NOTES.txt
|
||
</span></span><span style="display:flex;"><span>│ └── service.yaml
|
||
</span></span><span style="display:flex;"><span>└── values.yaml
|
||
</span></span></code></pre></div><ul>
|
||
<li>Chart.yaml 用于描述这个chart，包括名字，描述信息以及版本。</li>
|
||
<li>values.yaml 用于存储templates目录中模板文件中用到的变量。 模板文件一般是Go模板。如果你需要了解更多关于Go模板的相关信息，可以查看Hugo (<a href="https://gohugo.io">https://gohugo.io</a>) 的一个关于Go模板的介绍 (<a href="https://gohugo.io/templates/go-templates/">https://gohugo.io/templates/go-templates/</a>)。</li>
|
||
<li>NOTES.txt 用于向部署该chart的用于介绍chart部署后的一些信息。例如介绍如何使用这个chart，列出缺省的设置等。</li>
|
||
</ul>
|
||
<p>打开Chart.yaml, 填写你部署的应用的详细信息，以testapi为例：</p>
|
||
<pre tabindex="0"><code>apiVersion: v1
|
||
description: A simple api for testing and debugging
|
||
name: testapi-chart
|
||
version: 0.0.1
|
||
</code></pre><p>然后打开并根据需要编辑values.yaml。下面是testapi应用的values.yaml文件内容。</p>
|
||
<pre tabindex="0"><code>replicaCount: 2
|
||
image:
|
||
repository: daemonza/testapi
|
||
tag: latest
|
||
pullPolicy: IfNotPresent
|
||
service:
|
||
name: testapi
|
||
type: ClusterIP
|
||
externalPort: 80
|
||
internalPort: 80
|
||
resources:
|
||
limits:
|
||
cpu: 100m
|
||
memory: 128Mi
|
||
requests:
|
||
cpu: 100m
|
||
memory: 128Mi
|
||
</code></pre><p>在 testapi_chart 目录下运行下面命令以对chart进行校验。</p>
|
||
<pre tabindex="0"><code>helm lint
|
||
==> Linting .
|
||
[INFO] Chart.yaml: icon is recommended
|
||
|
||
1 chart(s) linted, no failures
|
||
</code></pre><p>如果文件格式错误，可以根据提示进行修改；如果一切正常，可以使用下面的命令对chart进行打包：</p>
|
||
<pre tabindex="0"><code>helm package testapi-chart --debug
|
||
</code></pre><p>这里添加了 –debug 参数来查看打包的输出，输出应该类似于：</p>
|
||
<pre tabindex="0"><code>Saved /Users/daemonza/testapi/testapi-chart/testapi-chart-0.0.1.tgz to current directory
|
||
Saved /Users/daemonza/testapi/testapi-chart/testapi-chart-0.0.1.tgz to /Users/daemonza/.helm/repository/local
|
||
</code></pre><p>chart被打包为一个压缩包testapi-chart-0.0.1.tgz，该压缩包被放到了当前目录下，并同时被保存到了helm的本地缺省仓库目录中。</p>
|
||
<h2 id="helm-repository">Helm Repository</h2>
|
||
<hr>
|
||
<p>虽然我们已经打包了chart并发布到了helm的本地目录中，但通过Helm search命令查找，并不能找不到刚才生成的chart包。</p>
|
||
<pre tabindex="0"><code>helm search testapi
|
||
No results found
|
||
</code></pre><p>这是因为repository目录中的chart还没有被Helm管理。我们可以在本地启动一个Repository Server，并将其加入到Helm repo列表中。</p>
|
||
<p>通过helm repo list命令可以看到目前helm中只配置了一个名为stable的repo，该repo指向了google的一个服务器。</p>
|
||
<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#282a36;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-text-size-adjust:none;"><code class="language-Bash" data-lang="Bash"><span style="display:flex;"><span>helm repo list
|
||
</span></span><span style="display:flex;"><span>NAME URL
|
||
</span></span><span style="display:flex;"><span>stable https://kubernetes-charts.storage.googleapis.com
|
||
</span></span></code></pre></div><p>使用helm serve命令启动一个repo server，该server缺省使用’$HELM_HOME/repository/local’目录作为chart存储，并在8879端口上提供服务。</p>
|
||
<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#282a36;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-text-size-adjust:none;"><code class="language-Bash" data-lang="Bash"><span style="display:flex;"><span>helm serve&
|
||
</span></span><span style="display:flex;"><span>Now serving you on 127.0.0.1:8879
|
||
</span></span></code></pre></div><p>启动本地repo server后，将其加入helm的repo列表。</p>
|
||
<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#282a36;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-text-size-adjust:none;"><code class="language-Bash" data-lang="Bash"><span style="display:flex;"><span>helm repo add <span style="color:#8be9fd;font-style:italic">local</span> http://127.0.0.1:8879
|
||
</span></span><span style="display:flex;"><span><span style="color:#f1fa8c">"local"</span> has been added to your repositories
|
||
</span></span></code></pre></div><p>现在再查找testapi chart包，就可以找到了。</p>
|
||
<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#282a36;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-text-size-adjust:none;"><code class="language-Bash" data-lang="Bash"><span style="display:flex;"><span>helm search testapi
|
||
</span></span><span style="display:flex;"><span>
|
||
</span></span><span style="display:flex;"><span>NAME CHART VERSION APP VERSION DESCRIPTION
|
||
</span></span><span style="display:flex;"><span>local/testapi-chart 0.0.1 A Helm chart <span style="color:#ff79c6">for</span> Kubernetes
|
||
</span></span></code></pre></div><h2 id="在kubernetes中部署chart">在kubernetes中部署Chart</h2>
|
||
<hr>
|
||
<p>chart被发布到仓储后，可以通过Helm instal命令部署chart，部署时指定chart名及Release（部署的实例）名：</p>
|
||
<pre tabindex="0"><code> helm install local/testapi-chart --name testapi
|
||
</code></pre><p>该命令的输出应类似:</p>
|
||
<pre tabindex="0"><code>NAME: testapi
|
||
LAST DEPLOYED: Mon Apr 16 10:21:44 2018
|
||
NAMESPACE: default
|
||
STATUS: DEPLOYED
|
||
|
||
RESOURCES:
|
||
==> v1/Service
|
||
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
|
||
testapi-testapi-chart ClusterIP 10.43.121.84 <none> 80/TCP 0s
|
||
|
||
==> v1beta1/Deployment
|
||
NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
|
||
testapi-testapi-chart 1 1 1 0 0s
|
||
|
||
==> v1/Pod(related)
|
||
NAME READY STATUS RESTARTS AGE
|
||
testapi-testapi-chart-9897d9f8c-nn6wd 0/1 Pending 0 0s
|
||
|
||
|
||
NOTES:
|
||
1. Get the application URL by running these commands:
|
||
export POD_NAME=$(kubectl get pods --namespace default -l "app=testapi-testapi-chart" -o jsonpath="{.items[0].metadata.name}")
|
||
echo "Visit http://127.0.0.1:8080 to use your application"
|
||
kubectl port-forward $POD_NAME 8080:80
|
||
</code></pre><p>使用下面的命令列出所有已部署的Release以及其对应的Chart。</p>
|
||
<pre tabindex="0"><code>helm ls
|
||
</code></pre><p>该命令的输出应类似:</p>
|
||
<pre tabindex="0"><code>NAME REVISION UPDATED STATUS CHART NAMESPACE
|
||
testapi 1 Mon Apr 16 10:21:44 2018 DEPLOYED testapi-chart-0.0.1 default
|
||
</code></pre><p>可以看到在输出中有一个Revision（更改历史）字段，该字段用于表示某一Release被更新的次数，可以用该特性对已部署的Release进行回滚。</p>
|
||
<h2 id="升级和回退">升级和回退</h2>
|
||
<hr>
|
||
<p>修改Chart.yaml，将版本号从0.0.1 修改为 1.0.0, 然后使用Helm package命令打包并发布到本地仓库。</p>
|
||
<p>查看本地库中的Chart信息，可以看到在本地仓库中testapi-chart有两个版本</p>
|
||
<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#282a36;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-text-size-adjust:none;"><code class="language-Bash" data-lang="Bash"><span style="display:flex;"><span>helm search testapi -l
|
||
</span></span><span style="display:flex;"><span>NAME CHART VERSION APP VERSION DESCRIPTION
|
||
</span></span><span style="display:flex;"><span>local/testapi-chart 0.0.1 A Helm chart <span style="color:#ff79c6">for</span> Kubernetes
|
||
</span></span><span style="display:flex;"><span>local/testapi-chart 1.0.0 A Helm chart <span style="color:#ff79c6">for</span> Kubernetes
|
||
</span></span></code></pre></div><p>现在用helm upgrade将已部署的testapi升级到新版本。可以通过参数指定需要升级的版本号，如果没有指定版本号，则缺省使用最新版本。</p>
|
||
<pre tabindex="0"><code>helm upgrade testapi local/testapi-chart
|
||
</code></pre><p>已部署的testapi release被升级到1.0.0版本</p>
|
||
<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#282a36;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-text-size-adjust:none;"><code class="language-Bash" data-lang="Bash"><span style="display:flex;"><span>helm list
|
||
</span></span><span style="display:flex;"><span>NAME REVISION UPDATED STATUS CHART NAMESPACE
|
||
</span></span><span style="display:flex;"><span>testapi <span style="color:#bd93f9">2</span> Mon Apr <span style="color:#bd93f9">16</span> 10:43:10 <span style="color:#bd93f9">2018</span> DEPLOYED testapi-chart-1.0.0 default
|
||
</span></span></code></pre></div><p>可以通过Helm history查看一个Release的多次更改。</p>
|
||
<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#282a36;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-text-size-adjust:none;"><code class="language-Bash" data-lang="Bash"><span style="display:flex;"><span>helm <span style="color:#8be9fd;font-style:italic">history</span> testapi
|
||
</span></span><span style="display:flex;"><span>REVISION UPDATED STATUS CHART DESCRIPTION
|
||
</span></span><span style="display:flex;"><span><span style="color:#bd93f9">1</span> Mon Apr <span style="color:#bd93f9">16</span> 10:21:44 <span style="color:#bd93f9">2018</span> SUPERSEDED testapi-chart-0.0.1 Install <span style="color:#8be9fd;font-style:italic">complete</span>
|
||
</span></span><span style="display:flex;"><span><span style="color:#bd93f9">2</span> Mon Apr <span style="color:#bd93f9">16</span> 10:43:10 <span style="color:#bd93f9">2018</span> DEPLOYED testapi-chart-1.0.0 Upgrade <span style="color:#8be9fd;font-style:italic">complete</span>
|
||
</span></span></code></pre></div><p>如果更新后的程序由于某些原因运行有问题，我们则需要回退到旧版本的应用，可以采用下面的命令进行回退。其中的参数1是前面Helm history中查看到的Release的更改历史。</p>
|
||
<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#282a36;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-text-size-adjust:none;"><code class="language-Bash" data-lang="Bash"><span style="display:flex;"><span>helm rollback testapi <span style="color:#bd93f9">1</span>
|
||
</span></span></code></pre></div><p>使用Helm list命令查看，部署的testapi的版本已经回退到0.0.1</p>
|
||
<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#282a36;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-text-size-adjust:none;"><code class="language-Bash" data-lang="Bash"><span style="display:flex;"><span>helm list
|
||
</span></span><span style="display:flex;"><span>NAME REVISION UPDATED STATUS CHART NAMESPACE
|
||
</span></span><span style="display:flex;"><span>testapi <span style="color:#bd93f9">3</span> Mon Apr <span style="color:#bd93f9">16</span> 10:48:20 <span style="color:#bd93f9">2018</span> DEPLOYED testapi-chart-0.0.1 default
|
||
</span></span></code></pre></div><h2 id="总结">总结</h2>
|
||
<hr>
|
||
<p>Helm作为kubernetes应用的包管理以及部署工具，提供了应用打包，发布，版本管理以及部署，升级，回退等功能。Helm以Chart软件包的形式简化Kubernetes的应用管理，提高了对用户的友好性。</p>
|
||
<h2 id="qa">Q&A</h2>
|
||
<hr>
|
||
<p>昨天在Docker.io技术微信群里面进行了Helm的分享，下面是分享过程中得到的一些有意思的反馈，进一步启发了我自己的一些思考。</p>
|
||
<p><strong>Q</strong>: Helm结合CD有什么好的建议吗？<BR>
|
||
<strong>A</strong>: 采用Helm可以把零散的Kubernetes应用配置文件作为一个chart管理，chart源码可以和源代码一起放到git库中管理。Helm还简了在CI/CD pipeline的软件部署流程。通过把chart参数化，可以在测试环境和生产环境可以采用不同的chart参数配置。</p>
|
||
<p>下图是采用了Helm的一个CI/CD流程
|
||
|
||
<img src="/img/2018-04-16-using-helm-to-deploy-to-kubernetes/ci-cd-jenkins-helm-k8s.png" alt="">
|
||
|
||
</p>
|
||
<p><strong>Q</strong>: 感谢分享，请问下多环境(test,staging，production)的业务配置如何管理呢？通过heml打包configmap吗，比如配置文件更新，也要重新打chats包吗？谢谢，这块我比较乱<BR>
|
||
<strong>A</strong>：Chart是支持参数替换的，可以把业务配置相关的参数设置为模板变量。使用Helm install Chart的时候可以指定一个参数值文件，这样就可以把业务参数从Chart中剥离了。例子： helm install –values=myvals.yaml wordpress</p>
|
||
<p><strong>Q</strong>: helm能解决服务依赖吗？<BR>
|
||
<strong>A</strong>：可以的，在chart可以通过requirements.yaml声明对其他chart的依赖关系。如下面声明表明chart依赖apache和mysql这两个第三方chart。</p>
|
||
<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#282a36;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-text-size-adjust:none;"><code class="language-yaml" data-lang="yaml"><span style="display:flex;"><span><span style="color:#ff79c6">dependencies</span>:
|
||
</span></span><span style="display:flex;"><span> - <span style="color:#ff79c6">name</span>: apache
|
||
</span></span><span style="display:flex;"><span> <span style="color:#ff79c6">version</span>: <span style="color:#bd93f9">1.2.3</span>
|
||
</span></span><span style="display:flex;"><span> <span style="color:#ff79c6">repository</span>: http://example.com/charts
|
||
</span></span><span style="display:flex;"><span> - <span style="color:#ff79c6">name</span>: mysql
|
||
</span></span><span style="display:flex;"><span> <span style="color:#ff79c6">version</span>: <span style="color:#bd93f9">3.2.1</span>
|
||
</span></span><span style="display:flex;"><span> <span style="color:#ff79c6">repository</span>: http://another.example.com/charts
|
||
</span></span></code></pre></div><p><strong>Q</strong>: chart的reversion 可以自定义吗？比如跟git的tag<BR>
|
||
<strong>A</strong>: 这位朋友应该是把chart的version和Release的reversion搞混了，呵呵。 Chart是没有reversion的，Chart部署的一个实例（Release）才有Reversion，Reversion是Release被更新后自动生成的。</p>
|
||
<p><strong>Q</strong>: 没有看到helm指向k8s的配置，怎么确认在哪个K8s集群运行的？<BR>
|
||
<strong>A</strong>: 使用和kubectl相同的配置，在 ~/.kube/config 中。</p>
|
||
<p><strong>Q</strong>: 这个简单例子并没有看出 Helm 相比 kubectl 有哪些优势，可以简要说一下吗？<BR>
|
||
<strong>A</strong>： Helm将kubernetes应用作为一个软件包整体管理，例如一个应用可能有前端服务器，后端服务器，数据库，这样会涉及多个Kubernetes 部署配置文件，Helm就整体管理了。另外Helm还提供了软件包版本，一键安装，升级，回退。Kubectl和Helm就好比你手工下载安装一个应用 和 使用apt-get 安装一个应用的区别。</p>
|
||
<p><strong>Q</strong>: 如何在helm install 时指定命名空间？<BR>
|
||
<strong>A</strong>: helm install local/testapi-chart –name testapi –namespace mynamespace</p>
|
||
<h2 id="参考">参考</h2>
|
||
<hr>
|
||
<ul>
|
||
<li><a href="https://daemonza.github.io/2017/02/20/using-helm-to-deploy-to-kubernetes/">Using Helm to deploy to Kubernetes</a></li>
|
||
<li><a href="https://docs.helm.sh/helm/">Helm documentation</a></li>
|
||
<li><a href="https://www.slideshare.net/alexLM/helm-application-deployment-management-for-kubernetes">Helm - Application deployment management for Kubernetes</a></li>
|
||
</ul>
|
||
|
||
|
||
|
||
|
||
<div class="entry-shang text-center">
|
||
|
||
<p>「真诚赞赏，手留余香」</p>
|
||
|
||
<button class="zs show-zs btn btn-bred">赞赏支持</button>
|
||
</div>
|
||
<div class="zs-modal-bg"></div>
|
||
<div class="zs-modal-box">
|
||
<div class="zs-modal-head">
|
||
<button type="button" class="close">×</button>
|
||
<span class="author"><a href="https://davidpaulyoung.com/"><img src="/img/favicon.png" />David Young</a></span>
|
||
|
||
<p class="tip"><i></i><span>真诚赞赏，手留余香</span></p>
|
||
|
||
|
||
</div>
|
||
<div class="zs-modal-body">
|
||
<div class="zs-modal-btns">
|
||
<button class="btn btn-blink" data-num="2">2元</button>
|
||
<button class="btn btn-blink" data-num="5">5元</button>
|
||
<button class="btn btn-blink" data-num="10">10元</button>
|
||
<button class="btn btn-blink" data-num="50">50元</button>
|
||
<button class="btn btn-blink" data-num="100">100元</button>
|
||
<button class="btn btn-blink" data-num="1">任意金额</button>
|
||
</div>
|
||
<div class="zs-modal-pay">
|
||
<button class="btn btn-bred" id="pay-text">2元</button>
|
||
<p>使用<span id="pay-type">微信</span>扫描二维码完成支付</p>
|
||
<img src="/img/reward/wechat-2.png" id="pay-image"/>
|
||
</div>
|
||
</div>
|
||
<div class="zs-modal-footer">
|
||
<label><input type="radio" name="zs-type" value="wechat" class="zs-type" checked="checked"><span ><span class="zs-wechat"><img src="/img/reward/wechat-btn.png"/></span></label>
|
||
<label><input type="radio" name="zs-type" value="alipay" class="zs-type" class="zs-alipay"><img src="/img/reward/alipay-btn.png"/></span></label>
|
||
</div>
|
||
</div>
|
||
<script type="text/javascript" src="/js/reward.js"></script>
|
||
|
||
|
||
|
||
|
||
<hr>
|
||
<ul class="pager">
|
||
|
||
<li class="previous">
|
||
<a href="/2018/04/11/service-mesh-vs-api-gateway/" data-toggle="tooltip" data-placement="top" title="Service Mesh 和 API Gateway的关系探讨（译文）">←
|
||
Previous Post</a>
|
||
</li>
|
||
|
||
|
||
<li class="next">
|
||
<a href="/2018/05/01/may-day-jiulonghu/" data-toggle="tooltip" data-placement="top" title="川西秘境探险">Next
|
||
Post →</a>
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link href="https://xxx.xxx.com/dist/Artalk.css" rel="stylesheet" />
|
||
<script src="https://xxx.xxx.com/dist/Artalk.js"></script>
|
||
|
||
|
||
<div id="Comments"></div>
|
||
|
||
<script>
|
||
Artalk.init({
|
||
el: '#Comments',
|
||
pageKey: 'https:\/\/davidpaulyoung.com\/2018\/04\/16\/using-helm-to-deploy-to-kubernetes\/',
|
||
pageTitle: 'Helm介绍',
|
||
server: 'https:\/\/xxx.xxx.com',
|
||
site: 'xxx blog',
|
||
})
|
||
</script>
|
||
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="
|
||
col-lg-2 col-lg-offset-0
|
||
visible-lg-block
|
||
sidebar-container
|
||
catalog-container">
|
||
<div class="side-catalog">
|
||
<hr class="hidden-sm hidden-xs">
|
||
<h5>
|
||
<a class="catalog-toggle" href="#">CATALOG</a>
|
||
</h5>
|
||
<ul class="catalog-body"></ul>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="
|
||
col-lg-8 col-lg-offset-2
|
||
col-md-10 col-md-offset-1
|
||
sidebar-container">
|
||
|
||
|
||
|
||
<section>
|
||
<hr class="hidden-sm hidden-xs">
|
||
<h5><a href="/tags/">FEATURED TAGS</a></h5>
|
||
<div class="tags">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<a href="/tags/docker" title="docker">
|
||
docker
|
||
</a>
|
||
|
||
|
||
|
||
|
||
|
||
<a href="/tags/istio" title="istio">
|
||
istio
|
||
</a>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<a href="/tags/kubernetes" title="kubernetes">
|
||
kubernetes
|
||
</a>
|
||
|
||
|
||
|
||
|
||
|
||
<a href="/tags/microservice" title="microservice">
|
||
microservice
|
||
</a>
|
||
|
||
|
||
|
||
|
||
|
||
<a href="/tags/security" title="security">
|
||
security
|
||
</a>
|
||
|
||
|
||
|
||
<a href="/tags/service-mesh" title="service mesh">
|
||
service mesh
|
||
</a>
|
||
|
||
|
||
|
||
<a href="/tags/tips" title="tips">
|
||
tips
|
||
</a>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</div>
|
||
</section>
|
||
|
||
|
||
|
||
|
||
<section>
|
||
<hr>
|
||
<h5>FRIENDS</h5>
|
||
<ul class="list-inline">
|
||
|
||
<li><a target="_blank" href="https://zhaozhihan.com">Linda的博客</a></li>
|
||
|
||
</ul>
|
||
</section>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
|
||
|
||
|
||
|
||
<footer>
|
||
<div class="container">
|
||
<div class="row">
|
||
<div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
|
||
<ul class="list-inline text-center">
|
||
|
||
<li>
|
||
<a href="mailto:youremail@gmail.com">
|
||
<span class="fa-stack fa-lg">
|
||
<i class="fas fa-circle fa-stack-2x"></i>
|
||
<i class="fas fa-envelope fa-stack-1x fa-inverse"></i>
|
||
</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<a target="_blank" href="/your%20wechat%20qr%20code%20image">
|
||
<span class="fa-stack fa-lg">
|
||
<i class="fas fa-circle fa-stack-2x"></i>
|
||
<i class="fab fa-weixin fa-stack-1x fa-inverse"></i>
|
||
</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li>
|
||
<a target="_blank" href="https://github.com/yourgithub">
|
||
<span class="fa-stack fa-lg">
|
||
<i class="fas fa-circle fa-stack-2x"></i>
|
||
<i class="fab fa-github fa-stack-1x fa-inverse"></i>
|
||
</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<a target="_blank" href="https://www.linkedin.com/in/yourlinkedinid">
|
||
<span class="fa-stack fa-lg">
|
||
<i class="fas fa-circle fa-stack-2x"></i>
|
||
<i class="fab fa-linkedin fa-stack-1x fa-inverse"></i>
|
||
</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
<li>
|
||
<a target="_blank" href="https://stackoverflow.com/users/yourstackoverflowid">
|
||
<span class="fa-stack fa-lg">
|
||
<i class="fas fa-circle fa-stack-2x"></i>
|
||
<i class="fab fa-stack-overflow fa-stack-1x fa-inverse"></i>
|
||
</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<a href='' rel="alternate" type="application/rss+xml" title="David Young" >
|
||
<span class="fa-stack fa-lg">
|
||
<i class="fas fa-circle fa-stack-2x"></i>
|
||
<i class="fas fa-rss fa-stack-1x fa-inverse"></i>
|
||
</span>
|
||
</a>
|
||
</li>
|
||
|
||
</ul>
|
||
<p class="copyright text-muted">
|
||
Copyright © David Young 2026
|
||
|
||
<br>
|
||
<a href="https://themes.gohugo.io/hugo-theme-cleanwhite">CleanWhite Hugo Theme</a> by <a href="https://zhaohuabing.com">Huabing</a> |
|
||
<iframe
|
||
style="margin-left: 2px; margin-bottom:-5px;"
|
||
frameborder="0" scrolling="0" width="100px" height="20px"
|
||
src="https://ghbtns.com/github-btn.html?user=zhaohuabing&repo=hugo-theme-cleanwhite&type=star&count=true" >
|
||
</iframe>
|
||
|
||
</p>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
|
||
|
||
<script>
|
||
function loadAsync(u, c) {
|
||
var d = document, t = 'script',
|
||
o = d.createElement(t),
|
||
s = d.getElementsByTagName(t)[0];
|
||
o.src = u;
|
||
if (c) { o.addEventListener('load', function (e) { c(null, e); }, false); }
|
||
s.parentNode.insertBefore(o, s);
|
||
}
|
||
</script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script>
|
||
|
||
if($('#tag_cloud').length !== 0){
|
||
loadAsync("/js/jquery.tagcloud.js",function(){
|
||
$.fn.tagcloud.defaults = {
|
||
|
||
color: {start: '#bbbbee', end: '#0085a1'},
|
||
};
|
||
$('#tag_cloud a').tagcloud();
|
||
})
|
||
}
|
||
</script>
|
||
|
||
|
||
<script>
|
||
(function() {
|
||
function updateTagcloudColors() {
|
||
const isDark = document.documentElement.getAttribute('data-theme') === 'dark';
|
||
const startColor = isDark ? '#808080' : '#bbbbee';
|
||
if($('#tag_cloud').length !== 0 && $.fn.tagcloud) {
|
||
$.fn.tagcloud.defaults = {
|
||
color: {start: startColor, end: '#0085a1'},
|
||
};
|
||
$('#tag_cloud a').tagcloud();
|
||
}
|
||
}
|
||
|
||
|
||
$(document).ready(function() {
|
||
updateTagcloudColors();
|
||
});
|
||
|
||
|
||
const observer = new MutationObserver(function(mutations) {
|
||
mutations.forEach(function(mutation) {
|
||
if (mutation.attributeName === 'data-theme') {
|
||
updateTagcloudColors();
|
||
}
|
||
});
|
||
});
|
||
|
||
observer.observe(document.documentElement, {
|
||
attributes: true,
|
||
attributeFilter: ['data-theme']
|
||
});
|
||
})();
|
||
</script>
|
||
|
||
|
||
<script>
|
||
loadAsync("https://cdn.jsdelivr.net/npm/fastclick@1.0.6/lib/fastclick.min.js", function(){
|
||
var $nav = document.querySelector("nav");
|
||
if($nav) FastClick.attach($nav);
|
||
})
|
||
</script>
|
||
|
||
|
||
<script src="/js/theme-toggle.js"></script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script type="text/javascript">
|
||
function generateCatalog(selector) {
|
||
|
||
|
||
|
||
|
||
|
||
_containerSelector = 'div.post-container'
|
||
|
||
|
||
|
||
var P = $(_containerSelector), a, n, t, l, i, c;
|
||
a = P.find('h1,h2,h3,h4,h5,h6');
|
||
|
||
|
||
$(selector).html('')
|
||
|
||
|
||
a.each(function () {
|
||
n = $(this).prop('tagName').toLowerCase();
|
||
i = "#" + $(this).prop('id');
|
||
t = $(this).text();
|
||
c = $('<a href="' + i + '" rel="nofollow" title="' + t + '">' + t + '</a>');
|
||
l = $('<li class="' + n + '_nav"></li>').append(c);
|
||
$(selector).append(l);
|
||
});
|
||
return true;
|
||
}
|
||
|
||
generateCatalog(".catalog-body");
|
||
|
||
|
||
$(".catalog-toggle").click((function (e) {
|
||
e.preventDefault();
|
||
$('.side-catalog').toggleClass("fold")
|
||
}))
|
||
|
||
|
||
|
||
|
||
loadAsync("\/js\/jquery.nav.js", function () {
|
||
$('.catalog-body').onePageNav({
|
||
currentClass: "active",
|
||
changeHash: !1,
|
||
easing: "swing",
|
||
filter: "",
|
||
scrollSpeed: 700,
|
||
scrollOffset: 0,
|
||
scrollThreshold: .2,
|
||
begin: null,
|
||
end: null,
|
||
scrollChange: null,
|
||
padding: 80
|
||
});
|
||
});
|
||
</script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</body>
|
||
</html>
|