How to use Nodes method of internal Package

...26	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"27	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/mock"28)29func createInternalLoadBalancer(gce *GCECloud, svc *v1.Service, existingFwdRule *compute.ForwardingRule, nodeNames []string, clusterName, clusterID, zoneName string) (*v1.LoadBalancerStatus, error) {30	nodes, err := createAndInsertNodes(gce, nodeNames, zoneName)31	if err != nil {32		return nil, err33	}34	return gce.ensureInternalLoadBalancer(35		clusterName,36		clusterID,37		svc,38		existingFwdRule,39		nodes,40	)41}42func TestEnsureInternalBackendServiceUpdates(t *testing.T) {43	t.Parallel()44	vals := DefaultTestClusterValues()45	nodeNames := []string{"test-node-1"}46	gce, err := fakeGCECloud(vals)47	require.NoError(t, err)48	svc := fakeLoadbalancerService(string(LBTypeInternal))49	lbName := cloudprovider.GetLoadBalancerName(svc)50	nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)51	igName := makeInstanceGroupName(vals.ClusterID)52	igLinks, err := gce.ensureInternalInstanceGroups(igName, nodes)53	require.NoError(t, err)54	sharedBackend := shareBackendService(svc)55	bsName := makeBackendServiceName(lbName, vals.ClusterID, sharedBackend, cloud.SchemeInternal, "TCP", svc.Spec.SessionAffinity)56	err = gce.ensureInternalBackendService(bsName, "description", svc.Spec.SessionAffinity, cloud.SchemeInternal, "TCP", igLinks, "")57	require.NoError(t, err)58	// Update the Internal Backend Service with a new ServiceAffinity59	err = gce.ensureInternalBackendService(bsName, "description", v1.ServiceAffinityNone, cloud.SchemeInternal, "TCP", igLinks, "")60	require.NoError(t, err)61	bs, err := gce.GetRegionBackendService(bsName, gce.region)62	assert.NoError(t, err)63	assert.Equal(t, bs.SessionAffinity, strings.ToUpper(string(v1.ServiceAffinityNone)))64}65func TestEnsureInternalBackendServiceGroups(t *testing.T) {66	for desc, tc := range map[string]struct {67		mockModifier func(*cloud.MockGCE)68	}{69		"Basic workflow": {},70		"GetRegionBackendService failed": {71			mockModifier: func(c *cloud.MockGCE) {72				c.MockRegionBackendServices.GetHook = mock.GetRegionBackendServicesErrHook73			},74		},75		"UpdateRegionBackendServices failed": {76			mockModifier: func(c *cloud.MockGCE) {77				c.MockRegionBackendServices.UpdateHook = mock.UpdateRegionBackendServicesErrHook78			},79		},80	} {81		t.Run(desc, func(t *testing.T) {82			t.Parallel()83			vals := DefaultTestClusterValues()84			nodeNames := []string{"test-node-1"}85			gce, err := fakeGCECloud(vals)86			require.NoError(t, err)87			svc := fakeLoadbalancerService(string(LBTypeInternal))88			lbName := cloudprovider.GetLoadBalancerName(svc)89			nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)90			igName := makeInstanceGroupName(vals.ClusterID)91			igLinks, err := gce.ensureInternalInstanceGroups(igName, nodes)92			require.NoError(t, err)93			sharedBackend := shareBackendService(svc)94			bsName := makeBackendServiceName(lbName, vals.ClusterID, sharedBackend, cloud.SchemeInternal, "TCP", svc.Spec.SessionAffinity)95			err = gce.ensureInternalBackendService(bsName, "description", svc.Spec.SessionAffinity, cloud.SchemeInternal, "TCP", igLinks, "")96			require.NoError(t, err)97			// Update the BackendService with new Instances98			if tc.mockModifier != nil {99				tc.mockModifier(gce.c.(*cloud.MockGCE))100			}101			newNodeNames := []string{"new-test-node-1", "new-test-node-2"}102			err = gce.ensureInternalBackendServiceGroups(bsName, newNodeNames)103			if tc.mockModifier != nil {104				assert.Error(t, err)105				return106			}107			assert.NoError(t, err)108			bs, err := gce.GetRegionBackendService(bsName, gce.region)109			assert.NoError(t, err)110			// Check that the instances are updated111			newNodes, err := createAndInsertNodes(gce, newNodeNames, vals.ZoneName)112			newIgLinks, err := gce.ensureInternalInstanceGroups(igName, newNodes)113			backends := backendsFromGroupLinks(newIgLinks)114			assert.Equal(t, bs.Backends, backends)115		})116	}117}118func TestEnsureInternalLoadBalancer(t *testing.T) {119	t.Parallel()120	vals := DefaultTestClusterValues()121	nodeNames := []string{"test-node-1"}122	gce, err := fakeGCECloud(vals)123	require.NoError(t, err)124	svc := fakeLoadbalancerService(string(LBTypeInternal))125	status, err := createInternalLoadBalancer(gce, svc, nil, nodeNames, vals.ClusterName, vals.ClusterID, vals.ZoneName)126	assert.NoError(t, err)127	assert.NotEmpty(t, status.Ingress)128	assertInternalLbResources(t, gce, svc, vals, nodeNames)129}130func TestEnsureInternalLoadBalancerWithExistingResources(t *testing.T) {131	t.Parallel()132	vals := DefaultTestClusterValues()133	nodeNames := []string{"test-node-1"}134	gce, err := fakeGCECloud(vals)135	require.NoError(t, err)136	svc := fakeLoadbalancerService(string(LBTypeInternal))137	// Create the expected resources necessary for an Internal Load Balancer138	nm := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}139	lbName := cloudprovider.GetLoadBalancerName(svc)140	sharedHealthCheck := !v1_service.RequestsOnlyLocalTraffic(svc)141	hcName := makeHealthCheckName(lbName, vals.ClusterID, sharedHealthCheck)142	hcPath, hcPort := GetNodesHealthCheckPath(), GetNodesHealthCheckPort()143	existingHC := newInternalLBHealthCheck(hcName, nm, sharedHealthCheck, hcPath, hcPort)144	err = gce.CreateHealthCheck(existingHC)145	require.NoError(t, err)146	nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)147	igName := makeInstanceGroupName(vals.ClusterID)148	igLinks, err := gce.ensureInternalInstanceGroups(igName, nodes)149	require.NoError(t, err)150	sharedBackend := shareBackendService(svc)151	bsDescription := makeBackendServiceDescription(nm, sharedBackend)152	bsName := makeBackendServiceName(lbName, vals.ClusterID, sharedBackend, cloud.SchemeInternal, "TCP", svc.Spec.SessionAffinity)153	err = gce.ensureInternalBackendService(bsName, bsDescription, svc.Spec.SessionAffinity, cloud.SchemeInternal, "TCP", igLinks, existingHC.SelfLink)154	require.NoError(t, err)155	_, err = createInternalLoadBalancer(gce, svc, nil, nodeNames, vals.ClusterName, vals.ClusterID, vals.ZoneName)156	assert.NoError(t, err)157}158func TestEnsureInternalLoadBalancerClearPreviousResources(t *testing.T) {159	t.Parallel()160	vals := DefaultTestClusterValues()161	gce, err := fakeGCECloud(vals)162	require.NoError(t, err)163	svc := fakeLoadbalancerService(string(LBTypeInternal))164	lbName := cloudprovider.GetLoadBalancerName(svc)165	// Create a ForwardingRule that's missing an IP address166	existingFwdRule := &compute.ForwardingRule{167		Name:                lbName,168		IPAddress:           "",169		Ports:               []string{"123"},170		IPProtocol:          "TCP",171		LoadBalancingScheme: string(cloud.SchemeInternal),172	}173	gce.CreateRegionForwardingRule(existingFwdRule, gce.region)174	// Create a Firewall that's missing a Description175	existingFirewall := &compute.Firewall{176		Name:    lbName,177		Network: gce.networkURL,178		Allowed: []*compute.FirewallAllowed{179			{180				IPProtocol: "tcp",181				Ports:      []string{"123"},182			},183		},184	}185	gce.CreateFirewall(existingFirewall)186	sharedHealthCheck := !v1_service.RequestsOnlyLocalTraffic(svc)187	hcName := makeHealthCheckName(lbName, vals.ClusterID, sharedHealthCheck)188	hcPath, hcPort := GetNodesHealthCheckPath(), GetNodesHealthCheckPort()189	nm := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}190	// Create a healthcheck with an incorrect threshold191	existingHC := newInternalLBHealthCheck(hcName, nm, sharedHealthCheck, hcPath, hcPort)192	existingHC.HealthyThreshold = gceHcHealthyThreshold * 10193	gce.CreateHealthCheck(existingHC)194	// Create a backend Service that's missing Description and Backends195	sharedBackend := shareBackendService(svc)196	backendServiceName := makeBackendServiceName(lbName, vals.ClusterID, sharedBackend, cloud.SchemeInternal, "TCP", svc.Spec.SessionAffinity)197	existingBS := &compute.BackendService{198		Name:                lbName,199		Protocol:            "TCP",200		HealthChecks:        []string{existingHC.SelfLink},201		SessionAffinity:     translateAffinityType(svc.Spec.SessionAffinity),202		LoadBalancingScheme: string(cloud.SchemeInternal),203	}204	gce.CreateRegionBackendService(existingBS, gce.region)205	existingFwdRule.BackendService = existingBS.Name206	_, err = createInternalLoadBalancer(gce, svc, existingFwdRule, []string{"test-node-1"}, vals.ClusterName, vals.ClusterID, vals.ZoneName)207	assert.NoError(t, err)208	// Expect new resources with the correct attributes to be created209	rule, _ := gce.GetRegionForwardingRule(lbName, gce.region)210	assert.NotEqual(t, existingFwdRule, rule)211	firewall, err := gce.GetFirewall(lbName)212	require.NoError(t, err)213	assert.NotEqual(t, firewall, existingFirewall)214	healthcheck, err := gce.GetHealthCheck(hcName)215	require.NoError(t, err)216	assert.NotEqual(t, healthcheck, existingHC)217	bs, err := gce.GetRegionBackendService(backendServiceName, gce.region)218	require.NoError(t, err)219	assert.NotEqual(t, bs, existingBS)220}221func TestUpdateInternalLoadBalancerBackendServices(t *testing.T) {222	t.Parallel()223	vals := DefaultTestClusterValues()224	nodeName := "test-node-1"225	gce, err := fakeGCECloud(vals)226	require.NoError(t, err)227	svc := fakeLoadbalancerService(string(LBTypeInternal))228	_, err = createInternalLoadBalancer(gce, svc, nil, []string{"test-node-1"}, vals.ClusterName, vals.ClusterID, vals.ZoneName)229	assert.NoError(t, err)230	// BackendService exists prior to updateInternalLoadBalancer call, but has231	// incorrect (missing) attributes.232	// ensureInternalBackendServiceGroups is called and creates the correct233	// BackendService234	lbName := cloudprovider.GetLoadBalancerName(svc)235	sharedBackend := shareBackendService(svc)236	backendServiceName := makeBackendServiceName(lbName, vals.ClusterID, sharedBackend, cloud.SchemeInternal, "TCP", svc.Spec.SessionAffinity)237	existingBS := &compute.BackendService{238		Name:                backendServiceName,239		Protocol:            "TCP",240		SessionAffinity:     translateAffinityType(svc.Spec.SessionAffinity),241		LoadBalancingScheme: string(cloud.SchemeInternal),242	}243	gce.CreateRegionBackendService(existingBS, gce.region)244	nodes, err := createAndInsertNodes(gce, []string{nodeName}, vals.ZoneName)245	require.NoError(t, err)246	err = gce.updateInternalLoadBalancer(vals.ClusterName, vals.ClusterID, svc, nodes)247	assert.NoError(t, err)248	bs, err := gce.GetRegionBackendService(backendServiceName, gce.region)249	require.NoError(t, err)250	// Check that the new BackendService has the correct attributes251	url_base := fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s", vals.ProjectID)252	assert.NotEqual(t, existingBS, bs)253	assert.Equal(254		t,255		bs.SelfLink,256		fmt.Sprintf("%s/regions/%s/backendServices/%s", url_base, vals.Region, bs.Name),257	)258	assert.Equal(t, bs.Description, `{"kubernetes.io/service-name":"/"}`)259	assert.Equal(260		t,261		bs.HealthChecks,262		[]string{fmt.Sprintf("%s/healthChecks/k8s-%s-node", url_base, vals.ClusterID)},263	)264}265func TestUpdateInternalLoadBalancerNodes(t *testing.T) {266	t.Parallel()267	vals := DefaultTestClusterValues()268	gce, err := fakeGCECloud(vals)269	require.NoError(t, err)270	node1Name := []string{"test-node-1"}271	svc := fakeLoadbalancerService(string(LBTypeInternal))272	nodes, err := createAndInsertNodes(gce, node1Name, vals.ZoneName)273	require.NoError(t, err)274	_, err = gce.ensureInternalLoadBalancer(vals.ClusterName, vals.ClusterID, svc, nil, nodes)275	assert.NoError(t, err)276	// Replace the node in initial zone; add new node in a new zone.277	node2Name, node3Name := "test-node-2", "test-node-3"278	newNodesZoneA, err := createAndInsertNodes(gce, []string{node2Name}, vals.ZoneName)279	require.NoError(t, err)280	newNodesZoneB, err := createAndInsertNodes(gce, []string{node3Name}, vals.SecondaryZoneName)281	require.NoError(t, err)282	nodes = append(newNodesZoneA, newNodesZoneB...)283	err = gce.updateInternalLoadBalancer(vals.ClusterName, vals.ClusterID, svc, nodes)284	assert.NoError(t, err)285	lbName := cloudprovider.GetLoadBalancerName(svc)286	sharedBackend := shareBackendService(svc)287	backendServiceName := makeBackendServiceName(lbName, vals.ClusterID, sharedBackend, cloud.SchemeInternal, "TCP", svc.Spec.SessionAffinity)288	bs, err := gce.GetRegionBackendService(backendServiceName, gce.region)289	require.NoError(t, err)290	assert.Equal(t, 2, len(bs.Backends), "Want two backends referencing two instances groups")291	for _, zone := range []string{vals.ZoneName, vals.SecondaryZoneName} {292		var found bool293		for _, be := range bs.Backends {294			if strings.Contains(be.Group, zone) {295				found = true296				break297			}298		}299		assert.True(t, found, "Expected list of backends to have zone %q", zone)300	}301	// Expect initial zone to have test-node-2302	igName := makeInstanceGroupName(vals.ClusterID)303	instances, err := gce.ListInstancesInInstanceGroup(igName, vals.ZoneName, "ALL")304	require.NoError(t, err)305	assert.Equal(t, 1, len(instances))306	assert.Contains(307		t,308		instances[0].Instance,309		fmt.Sprintf("projects/%s/zones/%s/instances/%s", vals.ProjectID, vals.ZoneName, node2Name),310	)311	// Expect initial zone to have test-node-3312	instances, err = gce.ListInstancesInInstanceGroup(igName, vals.SecondaryZoneName, "ALL")313	require.NoError(t, err)314	assert.Equal(t, 1, len(instances))315	assert.Contains(316		t,317		instances[0].Instance,318		fmt.Sprintf("projects/%s/zones/%s/instances/%s", vals.ProjectID, vals.SecondaryZoneName, node3Name),319	)320}321func TestEnsureInternalLoadBalancerDeleted(t *testing.T) {322	t.Parallel()323	vals := DefaultTestClusterValues()324	gce, err := fakeGCECloud(vals)325	require.NoError(t, err)326	svc := fakeLoadbalancerService(string(LBTypeInternal))327	_, err = createInternalLoadBalancer(gce, svc, nil, []string{"test-node-1"}, vals.ClusterName, vals.ClusterID, vals.ZoneName)328	assert.NoError(t, err)329	err = gce.ensureInternalLoadBalancerDeleted(vals.ClusterName, vals.ClusterID, svc)330	assert.NoError(t, err)331	assertInternalLbResourcesDeleted(t, gce, svc, vals, true)332}333func TestEnsureInternalLoadBalancerDeletedTwiceDoesNotError(t *testing.T) {334	t.Parallel()335	vals := DefaultTestClusterValues()336	gce, err := fakeGCECloud(vals)337	require.NoError(t, err)338	svc := fakeLoadbalancerService(string(LBTypeInternal))339	_, err = createInternalLoadBalancer(gce, svc, nil, []string{"test-node-1"}, vals.ClusterName, vals.ClusterID, vals.ZoneName)340	assert.NoError(t, err)341	err = gce.ensureInternalLoadBalancerDeleted(vals.ClusterName, vals.ClusterID, svc)342	assert.NoError(t, err)343	// Deleting the loadbalancer and resources again should not cause an error.344	err = gce.ensureInternalLoadBalancerDeleted(vals.ClusterName, vals.ClusterID, svc)345	assert.NoError(t, err)346	assertInternalLbResourcesDeleted(t, gce, svc, vals, true)347}348func TestEnsureInternalLoadBalancerWithSpecialHealthCheck(t *testing.T) {349	vals := DefaultTestClusterValues()350	nodeName := "test-node-1"351	gce, err := fakeGCECloud(vals)352	require.NoError(t, err)353	healthCheckNodePort := int32(10101)354	svc := fakeLoadbalancerService(string(LBTypeInternal))355	svc.Spec.HealthCheckNodePort = healthCheckNodePort356	svc.Spec.Type = v1.ServiceTypeLoadBalancer357	svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal358	status, err := createInternalLoadBalancer(gce, svc, nil, []string{nodeName}, vals.ClusterName, vals.ClusterID, vals.ZoneName)359	assert.NoError(t, err)360	assert.NotEmpty(t, status.Ingress)361	loadBalancerName := cloudprovider.GetLoadBalancerName(svc)362	hc, err := gce.GetHealthCheck(loadBalancerName)363	assert.NoError(t, err)364	assert.NotNil(t, hc)365	assert.Equal(t, int64(healthCheckNodePort), hc.HttpHealthCheck.Port)366}367func TestClearPreviousInternalResources(t *testing.T) {368	// Configure testing environment.369	vals := DefaultTestClusterValues()370	svc := fakeLoadbalancerService(string(LBTypeInternal))371	loadBalancerName := cloudprovider.GetLoadBalancerName(svc)372	nm := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}373	gce, err := fakeGCECloud(vals)374	c := gce.c.(*cloud.MockGCE)375	require.NoError(t, err)376	hc_1, err := gce.ensureInternalHealthCheck("hc_1", nm, false, "healthz", 12345)377	require.NoError(t, err)378	hc_2, err := gce.ensureInternalHealthCheck("hc_2", nm, false, "healthz", 12346)379	require.NoError(t, err)380	err = gce.ensureInternalBackendService(svc.ObjectMeta.Name, "", svc.Spec.SessionAffinity, cloud.SchemeInternal, v1.ProtocolTCP, []string{}, "")381	require.NoError(t, err)382	backendSvc, err := gce.GetRegionBackendService(svc.ObjectMeta.Name, gce.region)383	backendSvc.HealthChecks = []string{hc_1.SelfLink, hc_2.SelfLink}384	c.MockRegionBackendServices.DeleteHook = mock.DeleteRegionBackendServicesErrHook385	c.MockHealthChecks.DeleteHook = mock.DeleteHealthChecksInternalErrHook386	gce.clearPreviousInternalResources(svc, loadBalancerName, backendSvc, "expectedBSName", "expectedHCName")387	backendSvc, err = gce.GetRegionBackendService(svc.ObjectMeta.Name, gce.region)388	assert.NoError(t, err)389	assert.NotNil(t, backendSvc, "BackendService should not be deleted when api is mocked out.")390	hc_1, err = gce.GetHealthCheck("hc_1")391	assert.NoError(t, err)392	assert.NotNil(t, hc_1, "HealthCheck should not be deleted when there are more than one healthcheck attached.")393	hc_2, err = gce.GetHealthCheck("hc_2")394	assert.NoError(t, err)395	assert.NotNil(t, hc_2, "HealthCheck should not be deleted when there are more than one healthcheck attached.")396	c.MockRegionBackendServices.DeleteHook = mock.DeleteRegionBackendServicesInUseErrHook397	backendSvc.HealthChecks = []string{hc_1.SelfLink}398	gce.clearPreviousInternalResources(svc, loadBalancerName, backendSvc, "expectedBSName", "expectedHCName")399	hc_1, err = gce.GetHealthCheck("hc_1")400	assert.NoError(t, err)401	assert.NotNil(t, hc_1, "HealthCheck should not be deleted when api is mocked out.")402	c.MockHealthChecks.DeleteHook = mock.DeleteHealthChecksInuseErrHook403	gce.clearPreviousInternalResources(svc, loadBalancerName, backendSvc, "expectedBSName", "expectedHCName")404	hc_1, err = gce.GetHealthCheck("hc_1")405	assert.NoError(t, err)406	assert.NotNil(t, hc_1, "HealthCheck should not be deleted when api is mocked out.")407	c.MockRegionBackendServices.DeleteHook = nil408	c.MockHealthChecks.DeleteHook = nil409	gce.clearPreviousInternalResources(svc, loadBalancerName, backendSvc, "expectedBSName", "expectedHCName")410	backendSvc, err = gce.GetRegionBackendService(svc.ObjectMeta.Name, gce.region)411	assert.Error(t, err)412	assert.Nil(t, backendSvc, "BackendService should be deleted.")413	hc_1, err = gce.GetHealthCheck("hc_1")414	assert.Error(t, err)415	assert.Nil(t, hc_1, "HealthCheck should be deleted.")416}417func TestEnsureInternalFirewallSucceedsOnXPN(t *testing.T) {418	gce, err := fakeGCECloud(DefaultTestClusterValues())419	require.NoError(t, err)420	vals := DefaultTestClusterValues()421	svc := fakeLoadbalancerService(string(LBTypeInternal))422	fwName := cloudprovider.GetLoadBalancerName(svc)423	c := gce.c.(*cloud.MockGCE)424	c.MockFirewalls.InsertHook = mock.InsertFirewallsUnauthorizedErrHook425	c.MockFirewalls.UpdateHook = mock.UpdateFirewallsUnauthorizedErrHook426	gce.onXPN = true427	require.True(t, gce.OnXPN())428	recorder := record.NewFakeRecorder(1024)429	gce.eventRecorder = recorder430	nodes, err := createAndInsertNodes(gce, []string{"test-node-1"}, vals.ZoneName)431	require.NoError(t, err)432	sourceRange := []string{"10.0.0.0/20"}433	gce.ensureInternalFirewall(434		svc,435		fwName,436		"A sad little firewall",437		sourceRange,438		[]string{"123"},439		v1.ProtocolTCP,440		nodes)441	require.Nil(t, err, "Should success when XPN is on.")442	checkEvent(t, recorder, FilewallChangeMsg, true)443	// Create a firewall.444	c.MockFirewalls.InsertHook = nil445	c.MockFirewalls.UpdateHook = nil446	gce.onXPN = false447	gce.ensureInternalFirewall(448		svc,449		fwName,450		"A sad little firewall",451		sourceRange,452		[]string{"123"},453		v1.ProtocolTCP,454		nodes)455	require.Nil(t, err)456	existingFirewall, err := gce.GetFirewall(fwName)457	require.Nil(t, err)458	require.NotNil(t, existingFirewall)459	gce.onXPN = true460	c.MockFirewalls.InsertHook = mock.InsertFirewallsUnauthorizedErrHook461	c.MockFirewalls.UpdateHook = mock.UpdateFirewallsUnauthorizedErrHook462	// Try to update the firewall just created.463	gce.ensureInternalFirewall(464		svc,465		fwName,466		"A happy little firewall",467		sourceRange,468		[]string{"123"},469		v1.ProtocolTCP,470		nodes)471	require.Nil(t, err, "Should success when XPN is on.")472	checkEvent(t, recorder, FilewallChangeMsg, true)473}474func TestEnsureLoadBalancerDeletedSucceedsOnXPN(t *testing.T) {475	vals := DefaultTestClusterValues()476	gce, err := fakeGCECloud(vals)477	c := gce.c.(*cloud.MockGCE)478	recorder := record.NewFakeRecorder(1024)479	gce.eventRecorder = recorder480	require.NoError(t, err)481	_, err = createInternalLoadBalancer(gce, fakeLoadbalancerService(string(LBTypeInternal)), nil, []string{"test-node-1"}, vals.ClusterName, vals.ClusterID, vals.ZoneName)482	assert.NoError(t, err)483	c.MockFirewalls.DeleteHook = mock.DeleteFirewallsUnauthorizedErrHook484	gce.onXPN = true485	err = gce.ensureInternalLoadBalancerDeleted(vals.ClusterName, vals.ClusterID, fakeLoadbalancerService(string(LBTypeInternal)))486	assert.NoError(t, err)487	checkEvent(t, recorder, FilewallChangeMsg, true)488}489func TestEnsureInternalInstanceGroupsDeleted(t *testing.T) {490	vals := DefaultTestClusterValues()491	gce, err := fakeGCECloud(vals)492	c := gce.c.(*cloud.MockGCE)493	recorder := record.NewFakeRecorder(1024)494	gce.eventRecorder = recorder495	require.NoError(t, err)496	igName := makeInstanceGroupName(vals.ClusterID)497	svc := fakeLoadbalancerService(string(LBTypeInternal))498	_, err = createInternalLoadBalancer(gce, svc, nil, []string{"test-node-1"}, vals.ClusterName, vals.ClusterID, vals.ZoneName)499	assert.NoError(t, err)500	c.MockZones.ListHook = mock.ListZonesInternalErrHook501	err = gce.ensureInternalLoadBalancerDeleted(igName, vals.ClusterID, svc)502	assert.Error(t, err, mock.InternalServerError)503	ig, err := gce.GetInstanceGroup(igName, vals.ZoneName)504	assert.NoError(t, err)505	assert.NotNil(t, ig)506	c.MockZones.ListHook = nil507	c.MockInstanceGroups.DeleteHook = mock.DeleteInstanceGroupInternalErrHook508	err = gce.ensureInternalInstanceGroupsDeleted(igName)509	assert.Error(t, err, mock.InternalServerError)510	ig, err = gce.GetInstanceGroup(igName, vals.ZoneName)511	assert.NoError(t, err)512	assert.NotNil(t, ig)513	c.MockInstanceGroups.DeleteHook = nil514	err = gce.ensureInternalInstanceGroupsDeleted(igName)515	assert.NoError(t, err)516	ig, err = gce.GetInstanceGroup(igName, vals.ZoneName)517	assert.Error(t, err)518	assert.Nil(t, ig)519}520type EnsureILBParams struct {521	clusterName     string522	clusterID       string523	service         *v1.Service524	existingFwdRule *compute.ForwardingRule525	nodes           []*v1.Node526}527// newEnsureILBParams is the constructor of EnsureILBParams.528func newEnsureILBParams(nodes []*v1.Node) *EnsureILBParams {529	vals := DefaultTestClusterValues()530	return &EnsureILBParams{531		vals.ClusterName,532		vals.ClusterID,533		fakeLoadbalancerService(string(LBTypeInternal)),534		nil,535		nodes,536	}537}538// TestEnsureInternalLoadBalancerErrors tests the function539// ensureInternalLoadBalancer, making sure the system won't panic when540// exceptions raised by gce.541func TestEnsureInternalLoadBalancerErrors(t *testing.T) {542	vals := DefaultTestClusterValues()543	var params *EnsureILBParams544	for desc, tc := range map[string]struct {545		adjustParams func(*EnsureILBParams)546		injectMock   func(*cloud.MockGCE)547	}{548		"Create internal instance groups failed": {549			injectMock: func(c *cloud.MockGCE) {550				c.MockInstanceGroups.GetHook = mock.GetInstanceGroupInternalErrHook551			},552		},553		"Invalid existing forwarding rules given": {554			adjustParams: func(params *EnsureILBParams) {555				params.existingFwdRule = &compute.ForwardingRule{BackendService: "badBackendService"}556			},557			injectMock: func(c *cloud.MockGCE) {558				c.MockRegionBackendServices.GetHook = mock.GetRegionBackendServicesErrHook559			},560		},561		"EnsureInternalBackendService failed": {562			injectMock: func(c *cloud.MockGCE) {563				c.MockRegionBackendServices.GetHook = mock.GetRegionBackendServicesErrHook564			},565		},566		"Create internal health check failed": {567			injectMock: func(c *cloud.MockGCE) {568				c.MockHealthChecks.GetHook = mock.GetHealthChecksInternalErrHook569			},570		},571		"Create firewall failed": {572			injectMock: func(c *cloud.MockGCE) {573				c.MockFirewalls.InsertHook = mock.InsertFirewallsUnauthorizedErrHook574			},575		},576		"Create region forwarding rule failed": {577			injectMock: func(c *cloud.MockGCE) {578				c.MockForwardingRules.InsertHook = mock.InsertForwardingRulesInternalErrHook579			},580		},581		"Get region forwarding rule failed": {582			injectMock: func(c *cloud.MockGCE) {583				c.MockForwardingRules.GetHook = mock.GetForwardingRulesInternalErrHook584			},585		},586		"Delete region forwarding rule failed": {587			adjustParams: func(params *EnsureILBParams) {588				params.existingFwdRule = &compute.ForwardingRule{BackendService: "badBackendService"}589			},590			injectMock: func(c *cloud.MockGCE) {591				c.MockForwardingRules.DeleteHook = mock.DeleteForwardingRuleErrHook592			},593		},594	} {595		t.Run(desc, func(t *testing.T) {596			gce, err := fakeGCECloud(DefaultTestClusterValues())597			nodes, err := createAndInsertNodes(gce, []string{"test-node-1"}, vals.ZoneName)598			require.NoError(t, err)599			params = newEnsureILBParams(nodes)600			if tc.adjustParams != nil {601				tc.adjustParams(params)602			}603			if tc.injectMock != nil {604				tc.injectMock(gce.c.(*cloud.MockGCE))605			}606			status, err := gce.ensureInternalLoadBalancer(607				params.clusterName,608				params.clusterID,609				params.service,610				params.existingFwdRule,611				params.nodes,...

...113			return nil, nil, errors.Wrapf(err, "fetching %s", table)114		}115	}116	{117		var nodes *serverpb.NodesResponse118		s := zc.clusterPrinter.start("requesting nodes")119		err := zc.runZipFn(ctx, s, func(ctx context.Context) error {120			nodes, err = zc.status.Nodes(ctx, &serverpb.NodesRequest{})121			return err122		})123		if cErr := zc.z.createJSONOrError(s, debugBase+"/nodes.json", nodes, err); cErr != nil {124			return nil, nil, cErr125		}126		// In case nodes came up back empty (the Nodes() RPC failed), we127		// still want to inspect the per-node endpoints on the head128		// node. As per the above, we were able to connect at least to129		// that.130		nodeList = []statuspb.NodeStatus{{Desc: roachpb.NodeDescriptor{131			NodeID:     firstNodeDetails.NodeID,132			Address:    firstNodeDetails.Address,133			SQLAddress: firstNodeDetails.SQLAddress,134		}}}135		if nodes != nil {136			// If the nodes were found, use that instead.137			nodeList = nodes.Nodes138		}139		// We'll want livenesses to decide whether a node is decommissioned.140		var lresponse *serverpb.LivenessResponse141		s = zc.clusterPrinter.start("requesting liveness")142		err = zc.runZipFn(ctx, s, func(ctx context.Context) error {143			lresponse, err = zc.admin.Liveness(ctx, &serverpb.LivenessRequest{})144			return err145		})146		if cErr := zc.z.createJSONOrError(s, livenessName+".json", nodes, err); cErr != nil {147			return nil, nil, cErr148		}149		livenessByNodeID = map[roachpb.NodeID]livenesspb.NodeLivenessStatus{}150		if lresponse != nil {151			livenessByNodeID = lresponse.Statuses...

1// Copyright © 2019 Developer Network, LLC2//3// This file is subject to the terms and conditions defined in4// file 'LICENSE', which is part of this source code package.5package graph6import (7	"errors"8	"fmt"9	"sync"10)11type Heap struct {12	internal []interface{}13	// kv mapping of a node value to the node pointer14	nodes map[interface{}]*Node15	// kv mapping of a node value to it's index in the internal array16	values map[interface{}]int17	size int18	once sync.Once19}20func (h *Heap) Print() {21	for i := 0; i < h.size; i++ {22		v := h.nodes[h.internal[i]]23		fmt.Printf("index: %v | value: %v | cost: %v\n", i, v.Value, v.Cost)24	}25}26func (h *Heap) init() {27	h.once.Do(func() {28		h.internal = make([]interface{}, 0)29		h.nodes = make(map[interface{}]*Node)30		h.values = make(map[interface{}]int)31	})32}33func (h *Heap) Swap(i, j int) {34	h.init()35	iv := h.internal[i]36	jv := h.internal[j]37	// Swap the internal values for the dictionary38	h.internal[i] = jv39	h.values[jv] = i40	h.internal[j] = iv41	h.values[iv] = j42}43func (h *Heap) Up(i int) {44	h.init()45	if h.size > 0 {46		j := 047		if i > 1 {48			j = i / 2 // Parent of i49		}50		if h.nodes[h.internal[i]].Cost < h.nodes[h.internal[j]].Cost {51			h.Swap(i, j)52			h.Up(j)53		}54	}55}56func (h *Heap) Down(i int) {57	h.init()58	if h.size > 0 {59		var j int60		if 2*i <= h.size {61			if 2*i < h.size {62				left := 2 * i63				right := (2 * i) + 164				// set j as left, unless right is lower cost65				j = left66				if h.nodes[h.internal[right]] != nil && h.nodes[h.internal[right]].Cost < h.nodes[h.internal[left]].Cost {67					j = right68				}69			} else {70				j = 2 * i71			}72			if h.nodes[h.internal[j]] != nil {73				if h.nodes[h.internal[i]] == nil || h.nodes[h.internal[j]].Cost < h.nodes[h.internal[i]].Cost {74					h.Swap(i, j)75					h.Down(j)76				}77			}78		}79	}80}81func (h *Heap) Insert(n *Node) (err error) {82	h.init()83	// Set the dictionary value84	h.nodes[n.Value] = n85	// Add the value to the internal array86	h.internal = append(h.internal, n.Value)87	// Increment the size88	h.size++89	// add the map entry for the new element's index90	h.values[n.Value] = h.size - 191	// Heap Up92	h.Up(h.size - 1)93	return err94}95func (h *Heap) Min() (index int, err error) {96	h.init()97	if h.size > 0 {98		// root node99		index = 0100	} else {101		// empty heap102		err = errors.New("empty heap")103	}104	return index, err105}106func (h *Heap) ExtractMin() (min *Node, err error) {107	h.init()108	var mini int109	if mini, err = h.Min(); err == nil {110		min = h.nodes[h.internal[mini]]111		h.Delete(mini)112	}113	return min, err114}115func (h *Heap) Delete(i int) (err error) {116	h.init()117	// Clean up maps118	v := h.internal[i]119	delete(h.nodes, v)120	delete(h.values, v)121	// Delete an entry from the internal slice while122	// maintaining it's ordering123	//copy(h.internal[i:], h.internal[i+1:])124	for index := i + 1; index < h.size; index++ {125		// Shift cells up the array126		v := h.internal[index]127		h.internal[index-1] = v128		h.values[v] = index - 1129		h.internal[index] = nil130	}131	// decrease the size now that the heap has been re-adjusted132	h.size--133	// h.internal[h.size-1] = ""134	// h.internal = h.internal[:len(h.internal)-1]135	// Reorder the heap from the root136	h.Down(0)137	return err138}139func (h *Heap) Find(value interface{}) (index int, err error) {140	return index, err141}142func (h *Heap) DeleteElem(value interface{}) (err error) {143	h.init()144	return err145}146func (h *Heap) ChangeCost(value interface{}, parent *Node, cost float64) {147	h.init()148	if h.nodes[value] != nil && cost < h.nodes[value].Cost {149		h.nodes[value].Parent = parent150		// Update the cost of the node151		h.nodes[value].Cost = cost152		// Move the node up the heap153		h.Up(h.values[value])154	}155}156func (h *Heap) Size() int {157	return h.size158}...

1import (2func main() {3	g := graph.New(5)4	g.Add(1, 2)5	g.Add(1, 3)6	g.Add(2, 3)7	g.Add(2, 4)8	g.Add(3, 4)9	g.Add(3, 5)10	g.Add(4, 5)11	fmt.Println(g.Nodes())12}13import (14func main() {15	g := graph.New(5)16	g.Add(1, 2)17	g.Add(1, 3)18	g.Add(2, 3)19	g.Add(2, 4)20	g.Add(3, 4)21	g.Add(3, 5)22	g.Add(4, 5)23	fmt.Println(g.Nodes())24}25import (26func main() {27	g := graph.New(5)28	g.Add(1, 2)29	g.Add(1, 3)30	g.Add(2, 3)31	g.Add(2, 4)32	g.Add(3, 4)33	g.Add(3, 5)34	g.Add(4, 5)35	fmt.Println(g.Nodes())36}

1import "fmt"2func main() {3    fmt.Println("Hello, playground")4    tree := NewTree()5    tree.Add(1)6    tree.Add(2)7    tree.Add(3)8    tree.Add(4)9    tree.Add(5)10    fmt.Println(tree.Nodes())11}12import "fmt"13func main() {14    fmt.Println("Hello, playground")15    tree := NewTree()16    tree.Add(1)17    tree.Add(2)18    tree.Add(3)19    tree.Add(4)20    tree.Add(5)21    fmt.Println(tree.Nodes())22}23import "fmt"24func main() {25    fmt.Println("Hello, playground")26    tree := NewTree()27    tree.Add(1)28    tree.Add(2)29    tree.Add(3)30    tree.Add(4)31    tree.Add(5)32    fmt.Println(tree.Nodes())33}34import "fmt"35func main() {36    fmt.Println("Hello, playground")37    tree := NewTree()38    tree.Add(1)39    tree.Add(2)40    tree.Add(3)41    tree.Add(4)42    tree.Add(5)43    fmt.Println(tree.Nodes())44}45import "fmt"46func main() {47    fmt.Println("Hello, playground")48    tree := NewTree()49    tree.Add(1)50    tree.Add(2)51    tree.Add(3)52    tree.Add(4)53    tree.Add(5)54    fmt.Println(tree.Nodes())55}56import "fmt"57func main() {58    fmt.Println("Hello, playground")59    tree := NewTree()60    tree.Add(1)61    tree.Add(2)62    tree.Add(3)63    tree.Add(4)64    tree.Add(5)65    fmt.Println(tree.Nodes())66}67import "fmt"68func main() {69    fmt.Println("Hello, playground")

1import "fmt"2type Node struct {3}4func main() {5    n := Node{Val: 1}6    fmt.Println(n.Nodes())7}8import "fmt"9type Node struct {10}11func (n *Node) Nodes() []*Node {12    return []*Node{n}13}14func main() {15    n := Node{Val: 1}16    fmt.Println(n.Nodes())17}

1import (2func main() {3    root = tree.Node{Value: 3}4    root.Left = &tree.Node{}5    root.Right = &tree.Node{5, nil, nil}6    root.Right.Left = new(tree.Node)7    root.Left.Right = tree.CreateNode(2)8    root.Right.Left.SetValue(4)9    root.Traverse()10    fmt.Println()11    myRoot := tree.Node{Value: 3}12    myRoot.Left = &tree.Node{}13    myRoot.Right = &tree.Node{5, nil, nil}14    myRoot.Right.Left = new(tree.Node)15    myRoot.Left.Right = tree.CreateNode(2)16    myRoot.Right.Left.SetValue(4)17    myRoot.Traverse()18    fmt.Println()19    nodes := []tree.Node{20        {Value: 3},21        {},22        {6, nil, &root},23    }24    fmt.Println(nodes)25}26import (27func main() {28    root = tree.Node{Value: 3}29    root.Left = &tree.Node{}30    root.Right = &tree.Node{5, nil, nil}31    root.Right.Left = new(tree.Node)32    root.Left.Right = tree.CreateNode(2)33    root.Right.Left.SetValue(4)34    root.Traverse()35    fmt.Println()36    myRoot := tree.Node{Value: 3}37    myRoot.Left = &tree.Node{}38    myRoot.Right = &tree.Node{5, nil, nil}39    myRoot.Right.Left = new(tree.Node)40    myRoot.Left.Right = tree.CreateNode(2)41    myRoot.Right.Left.SetValue(4)42    myRoot.Traverse()43    fmt.Println()44    nodes := []tree.Node{45        {Value: 3},46        {},47        {6, nil, &root},48    }49    fmt.Println(nodes)50}51import (52func main() {53    root = tree.Node{Value:

1import (2func main() {3    g := graph.NewAdjacencyList(4)4    g.AddEdge(0, 1)5    g.AddEdge(1, 2)6    g.AddEdge(2, 3)7    g.AddEdge(3, 0)8    g.AddEdge(0, 2)9    g.AddEdge(1, 3)10    fmt.Println(g.Nodes())11}12type Node struct {13}14type Edge struct {15}16import (17func main() {18    g := graph.NewAdjacencyList(4)19    g.AddEdge(0, 1)20    g.AddEdge(1, 2)21    g.AddEdge(2, 3)22    g.AddEdge(3, 0)23    g.AddEdge(0, 2)24    g.AddEdge(1, 3)25    fmt.Println(g.Nodes())26    fmt.Println(g.Edges())27    fmt.Println(g.Cost(0, 1))28    fmt.Println(g.Cost(1, 2))29    fmt.Println(g.Cost(2, 3))30    fmt.Println(g.Cost(3, 0))31    fmt.Println(g.Cost(0, 2))32    fmt.Println(g.Cost(1, 3))33}34[{0 [1 2]} {1 [2 3]} {2 [3

1import (2func main() {3    g := simple.NewDirectedGraph()4    g.AddNode(simple.Node(1))5    g.AddNode(simple.Node(2))6    g.AddNode(simple.Node(3))7    g.AddNode(simple.Node(4))8    g.AddNode(simple.Node(5))9    g.AddNode(simple.Node(6))10    g.AddNode(simple.Node(7))11    g.AddNode(simple.Node(8))12    g.AddNode(simple.Node(9))13    g.AddNode(simple.Node(10))14    g.SetEdge(simple.Edge{F: simple.Node(1), T: simple.Node(2)})15    g.SetEdge(simple.Edge{F: simple.Node(1), T: simple.Node(3)})16    g.SetEdge(simple.Edge{F: simple.Node(1), T: simple.Node(4)})17    g.SetEdge(simple.Edge{F: simple.Node(2), T: simple.Node(5)})18    g.SetEdge(simple.Edge{F: simple.Node(2), T: simple.Node(6)})19    g.SetEdge(simple.Edge{F: simple.Node(3), T: simple.Node(7)})20    g.SetEdge(simple.Edge{F: simple.Node(3), T: simple.Node(8)})21    g.SetEdge(simple.Edge{F: simple.Node(4), T: simple.Node(9)})22    g.SetEdge(simple.Edge{F: simple.Node(4), T: simple.Node(10)})23    fmt.Println(g.Nodes())24}

1import "internal/pkg1"2func main() {3    pkg1.NewNode().Nodes()4}5type node struct {}6func NewNode() *node {7    return &node{}8}9func (n *node) Nodes() {10    n.nodes()11}12func (n *node) nodes() {13    println("nodes")14}15import "internal/pkg1"16func main() {17    pkg1.NewNode().Nodes()18}19type node struct {}20func NewNode() *node {21    return &node{}22}23func (n *node) Nodes() {24    n.nodes()25}26func (n *node) nodes() {27    println("nodes")28}29func (n *node) nodes() {30    println("nodes2")31}

1import (2func main() {3}4import (5func main() {6}7import (8func main() {9}10import (11func main() {12}13import (14func main() {15}16import (17func main() {18}19import (20func main() {21}22import (23func main() {24}